package pipeline import ( "context" "crypto/rand" "database/sql" "encoding/hex" "fmt" "time" ) const SessionExportJobAttempts = 3 // SessionExportJob is an immutable ownership snapshot plus durable recovery // state. Attempts count recovery operations, not the export CLI's inner reads. type SessionExportJob struct { RunID int64 `json:"run_id"` Sequence int `json:"sequence"` SessionID string `json:"session_id"` Backend string `json:"backend"` Directory string `json:"directory"` Reference string `json:"-"` Status string `json:"status"` Attempts int `json:"attempts"` NextAt int64 `json:"next_attempt_at_ms"` Diagnostic string `json:"diagnostic,omitempty"` Token string `json:"-"` } func (s *Store) initSessionExportJobs() error { _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_jobs ( run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, session_id TEXT NOT NULL, backend TEXT NOT NULL, directory TEXT NOT NULL, reference TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','complete','exhausted')), attempts INTEGER NOT NULL DEFAULT 0 CHECK(attempts BETWEEN 0 AND 3), next_at INTEGER NOT NULL, lease_until INTEGER NOT NULL DEFAULT 0, token TEXT NOT NULL DEFAULT '', diagnostic TEXT NOT NULL DEFAULT '', PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE); CREATE INDEX IF NOT EXISTS session_export_jobs_due ON session_export_jobs(status,next_at,lease_until);`) if err == nil { _, err = s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_gates ( run_id INTEGER PRIMARY KEY, sequence INTEGER NOT NULL, FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE)`) } return err } // Missing or incomplete original evidence is recoverable only with an exact // native ownership record. No backend guesses, prompt parsing or stage reruns. const enqueueExportJobs = `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at) SELECT s.run_id,s.sequence,s.session_id,o.backend,r.workspace,o.reference,? FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence LEFT JOIN step_session_exports x ON x.run_id=s.run_id AND x.sequence=s.sequence WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND (o.backend IN ('v1','v2') OR (o.backend='pi' AND o.reference<>'')) AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=s.run_id AND j.sequence=s.sequence) AND NOT (COALESCE(json_extract(x.export_json,'$.status'),'complete')='complete' AND COALESCE(json_extract(x.export_json,'$.session_id'),'')=s.session_id AND COALESCE(json_extract(x.export_json,'$.backend'),'')=o.backend AND COALESCE(json_extract(x.export_json,'$.messages'),0)>0 AND COALESCE(json_extract(x.export_json,'$.html'),'')<>'' AND COALESCE(json_extract(x.export_json,'$.html_bytes'),0)>0 AND COALESCE(json_extract(x.export_json,'$.truncated'),0)=0 AND COALESCE(json_extract(x.export_json,'$.diagnostic'),'')='')` func enqueueStepExportJob(ctx context.Context, tx *sql.Tx, id int64, sequence int) error { _, err := tx.ExecContext(ctx, enqueueExportJobs+` AND s.run_id=? AND s.sequence=? ON CONFLICT(run_id,sequence) DO NOTHING`, time.Now().Add(5*time.Second).UnixMilli(), id, sequence) return err } // ReconcileSessionExportJobs recovers interrupted initial capture and historical // missing jobs from source-of-truth step ownership. Existing schedules, exhausted // budgets and completed jobs are never reset, even across repeated restarts. func (s *Store) ReconcileSessionExportJobs(ctx context.Context, now time.Time) error { _, err := s.db.ExecContext(ctx, enqueueExportJobs+` LIMIT 256 ON CONFLICT(run_id,sequence) DO NOTHING`, now.Add(5*time.Second).UnixMilli()) return err } const exactExportJobOwner = `EXISTS(SELECT 1 FROM run_steps s JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence JOIN pipeline_runs r ON r.id=s.run_id WHERE s.run_id=j.run_id AND s.sequence=j.sequence AND s.session_id=j.session_id AND o.backend=j.backend AND o.reference=j.reference AND r.workspace=j.directory)` // A complete repair must match the job's identity and have nonempty verified // file evidence. This also closes the crash window after repair persistence but // before acknowledgement of its job lease, without writing another export. const completeExportJobEvidence = `EXISTS(SELECT 1 FROM session_export_repairs p WHERE p.run_id=j.run_id AND p.sequence=j.sequence AND json_extract(p.export_json,'$.status')='complete' AND json_extract(p.export_json,'$.session_id')=j.session_id AND json_extract(p.export_json,'$.backend')=j.backend AND json_extract(p.export_json,'$.directory')=j.directory AND json_extract(p.export_json,'$.messages')>0 AND json_extract(p.export_json,'$.html_bytes')>0 AND COALESCE(json_extract(p.export_json,'$.html'),'')<>'' AND COALESCE(json_extract(p.export_json,'$.truncated'),0)=0 AND COALESCE(json_extract(p.export_json,'$.diagnostic'),'')='')` // cleanupGuard supports the older audit branch without editing its store or // cancellation schema. On current main the persisted flag is mandatory in the // claim predicate. The legacy schema has no cleanup-unconfirmed state. func (s *Store) exportJobCleanupGuard(ctx context.Context) (string, error) { rows, err := s.db.QueryContext(ctx, "PRAGMA table_info(pipeline_runs)") if err != nil { return "", err } defer rows.Close() for rows.Next() { var cid, notnull, pk int var name, kind string var def sql.NullString if err := rows.Scan(&cid, &name, &kind, ¬null, &def, &pk); err != nil { return "", err } if name == "cleanup_unconfirmed" { return " AND r.cleanup_unconfirmed=0", nil } } return "", rows.Err() } func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease time.Duration) (*SessionExportJob, error) { if lease <= 0 { return nil, fmt.Errorf("session export recovery lease must be positive") } guard, err := s.exportJobCleanupGuard(ctx) if err != nil { return nil, err } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return nil, err } defer tx.Rollback() // A persisted successful repair wins, including after a process restart. if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE status<>'complete' AND `+completeExportJobEvidence); err != nil { return nil, err } if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Original session ownership changed or is unavailable; automatic recovery refused' WHERE status IN ('pending','running') AND NOT `+exactExportJobOwner); err != nil { return nil, err } if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Recovery budget exhausted; last worker stopped before acknowledgement' WHERE (status='pending' OR (status='running' AND lease_until<=?)) AND attempts>=?`, now.UnixMilli(), SessionExportJobAttempts); err != nil { return nil, err } var job SessionExportJob err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts0) ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic) if err == sql.ErrNoRows { return nil, tx.Commit() } if err != nil { return nil, err } var nonce [16]byte if _, err := rand.Read(nonce[:]); err != nil { return nil, err } job.Token = hex.EncodeToString(nonce[:]) if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='running',attempts=attempts+1,token=?,lease_until=? WHERE run_id=? AND sequence=?`, job.Token, now.Add(lease).UnixMilli(), job.RunID, job.Sequence); err != nil { return nil, err } job.Attempts++ job.Status = "running" if err := tx.Commit(); err != nil { return nil, err } return &job, nil } // DeferSessionExportJob returns a lease without spending a retry when the // runner's in-memory cleanup gate is still held. No native operation ran. func (s *Store) DeferSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time) error { return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status='pending',attempts=attempts-1,token='',lease_until=0,next_at=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, now.Add(5*time.Second).UnixMilli(), job.RunID, job.Sequence, job.Token)) } func (s *Store) FinishSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time, diagnostic string) error { // Never trust an optimistic worker return as proof of a full export. result, err := s.db.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE run_id=? AND sequence=? AND status='running' AND token=? AND `+completeExportJobEvidence, job.RunID, job.Sequence, job.Token) if err != nil { return err } if n, _ := result.RowsAffected(); n == 1 { return nil } status := "pending" delay := 20 * time.Second if job.Attempts == 2 { delay = 60 * time.Second } if job.Attempts >= SessionExportJobAttempts { status = "exhausted" } nextAt := now.Add(delay).UnixMilli() if status == "exhausted" { nextAt = 0 } if diagnostic == "" { diagnostic = "Native session evidence remains incomplete; upstream recovery is not guaranteed" } if len(diagnostic) > 2048 { diagnostic = diagnostic[:2048] } return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status=?,token='',lease_until=0,next_at=?,diagnostic=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, status, nextAt, diagnostic, job.RunID, job.Sequence, job.Token)) } func (s *Store) SessionExportJob(ctx context.Context, id int64, sequence int) (*SessionExportJob, error) { var job SessionExportJob err := s.db.QueryRowContext(ctx, `SELECT run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, fmt.Errorf("session export recovery: %w", err) } return &job, nil }