Files
pipelines/internal/pipeline/session_export_jobs.go
T

214 lines
11 KiB
Go

package pipeline
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"fmt"
"time"
)
const SessionExportJobAttempts = 3
// SessionExportJob is an immutable ownership snapshot plus durable recovery
// state. Attempts count recovery operations, not the export CLI's inner reads.
type SessionExportJob struct {
RunID int64 `json:"run_id"`
Sequence int `json:"sequence"`
SessionID string `json:"session_id"`
Backend string `json:"backend"`
Directory string `json:"directory"`
Reference string `json:"-"`
Status string `json:"status"`
Attempts int `json:"attempts"`
NextAt int64 `json:"next_attempt_at_ms"`
Diagnostic string `json:"diagnostic,omitempty"`
Token string `json:"-"`
}
func (s *Store) initSessionExportJobs() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_jobs (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL,
session_id TEXT NOT NULL, backend TEXT NOT NULL, directory TEXT NOT NULL, reference TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','complete','exhausted')), attempts INTEGER NOT NULL DEFAULT 0 CHECK(attempts BETWEEN 0 AND 3),
next_at INTEGER NOT NULL, lease_until INTEGER NOT NULL DEFAULT 0, token TEXT NOT NULL DEFAULT '', diagnostic TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE);
CREATE INDEX IF NOT EXISTS session_export_jobs_due ON session_export_jobs(status,next_at,lease_until);`)
if err == nil {
_, err = s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_gates (
run_id INTEGER PRIMARY KEY, sequence INTEGER NOT NULL,
FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE)`)
}
return err
}
// Missing or incomplete original evidence is recoverable only with an exact
// native ownership record. No backend guesses, prompt parsing or stage reruns.
const enqueueExportJobs = `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at)
SELECT s.run_id,s.sequence,s.session_id,o.backend,r.workspace,o.reference,?
FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id
JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence
LEFT JOIN step_session_exports x ON x.run_id=s.run_id AND x.sequence=s.sequence
WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND (o.backend IN ('v1','v2') OR (o.backend='pi' AND o.reference<>''))
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=s.run_id AND j.sequence=s.sequence)
AND NOT (COALESCE(json_extract(x.export_json,'$.status'),'complete')='complete'
AND COALESCE(json_extract(x.export_json,'$.session_id'),'')=s.session_id
AND COALESCE(json_extract(x.export_json,'$.backend'),'')=o.backend
AND COALESCE(json_extract(x.export_json,'$.messages'),0)>0
AND COALESCE(json_extract(x.export_json,'$.html'),'')<>''
AND COALESCE(json_extract(x.export_json,'$.html_bytes'),0)>0
AND COALESCE(json_extract(x.export_json,'$.truncated'),0)=0
AND COALESCE(json_extract(x.export_json,'$.diagnostic'),'')='')`
func enqueueStepExportJob(ctx context.Context, tx *sql.Tx, id int64, sequence int) error {
_, err := tx.ExecContext(ctx, enqueueExportJobs+` AND s.run_id=? AND s.sequence=? ON CONFLICT(run_id,sequence) DO NOTHING`, time.Now().Add(5*time.Second).UnixMilli(), id, sequence)
return err
}
// ReconcileSessionExportJobs recovers interrupted initial capture and historical
// missing jobs from source-of-truth step ownership. Existing schedules, exhausted
// budgets and completed jobs are never reset, even across repeated restarts.
func (s *Store) ReconcileSessionExportJobs(ctx context.Context, now time.Time) error {
_, err := s.db.ExecContext(ctx, enqueueExportJobs+` LIMIT 256 ON CONFLICT(run_id,sequence) DO NOTHING`, now.Add(5*time.Second).UnixMilli())
return err
}
const exactExportJobOwner = `EXISTS(SELECT 1 FROM run_steps s JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence JOIN pipeline_runs r ON r.id=s.run_id WHERE s.run_id=j.run_id AND s.sequence=j.sequence AND s.session_id=j.session_id AND o.backend=j.backend AND o.reference=j.reference AND r.workspace=j.directory)`
// A complete repair must match the job's identity and have nonempty verified
// file evidence. This also closes the crash window after repair persistence but
// before acknowledgement of its job lease, without writing another export.
const completeExportJobEvidence = `EXISTS(SELECT 1 FROM session_export_repairs p WHERE p.run_id=j.run_id AND p.sequence=j.sequence
AND json_extract(p.export_json,'$.status')='complete'
AND json_extract(p.export_json,'$.session_id')=j.session_id
AND json_extract(p.export_json,'$.backend')=j.backend
AND json_extract(p.export_json,'$.directory')=j.directory
AND json_extract(p.export_json,'$.messages')>0 AND json_extract(p.export_json,'$.html_bytes')>0
AND COALESCE(json_extract(p.export_json,'$.html'),'')<>''
AND COALESCE(json_extract(p.export_json,'$.truncated'),0)=0
AND COALESCE(json_extract(p.export_json,'$.diagnostic'),'')='')`
// cleanupGuard supports the older audit branch without editing its store or
// cancellation schema. On current main the persisted flag is mandatory in the
// claim predicate. The legacy schema has no cleanup-unconfirmed state.
func (s *Store) exportJobCleanupGuard(ctx context.Context) (string, error) {
rows, err := s.db.QueryContext(ctx, "PRAGMA table_info(pipeline_runs)")
if err != nil {
return "", err
}
defer rows.Close()
for rows.Next() {
var cid, notnull, pk int
var name, kind string
var def sql.NullString
if err := rows.Scan(&cid, &name, &kind, &notnull, &def, &pk); err != nil {
return "", err
}
if name == "cleanup_unconfirmed" {
return " AND r.cleanup_unconfirmed=0", nil
}
}
return "", rows.Err()
}
func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease time.Duration) (*SessionExportJob, error) {
if lease <= 0 {
return nil, fmt.Errorf("session export recovery lease must be positive")
}
guard, err := s.exportJobCleanupGuard(ctx)
if err != nil {
return nil, err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return nil, err
}
defer tx.Rollback()
// A persisted successful repair wins, including after a process restart.
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE status<>'complete' AND `+completeExportJobEvidence); err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Original session ownership changed or is unavailable; automatic recovery refused' WHERE status IN ('pending','running') AND NOT `+exactExportJobOwner); err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Recovery budget exhausted; last worker stopped before acknowledgement' WHERE (status='pending' OR (status='running' AND lease_until<=?)) AND attempts>=?`, now.UnixMilli(), SessionExportJobAttempts); err != nil {
return nil, err
}
var job SessionExportJob
err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic
FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence
WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts<?
AND (r.status IN ('succeeded','failed','canceled','interrupted') OR (r.status='waiting' AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=j.run_id AND g.sequence=j.sequence))) AND s.status NOT IN ('running','waiting')`+guard+`
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=j.run_id OR a.session_id=j.session_id) AND a.agent_active<>0)
ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
if err == sql.ErrNoRows {
return nil, tx.Commit()
}
if err != nil {
return nil, err
}
var nonce [16]byte
if _, err := rand.Read(nonce[:]); err != nil {
return nil, err
}
job.Token = hex.EncodeToString(nonce[:])
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='running',attempts=attempts+1,token=?,lease_until=? WHERE run_id=? AND sequence=?`, job.Token, now.Add(lease).UnixMilli(), job.RunID, job.Sequence); err != nil {
return nil, err
}
job.Attempts++
job.Status = "running"
if err := tx.Commit(); err != nil {
return nil, err
}
return &job, nil
}
// DeferSessionExportJob returns a lease without spending a retry when the
// runner's in-memory cleanup gate is still held. No native operation ran.
func (s *Store) DeferSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time) error {
return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status='pending',attempts=attempts-1,token='',lease_until=0,next_at=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, now.Add(5*time.Second).UnixMilli(), job.RunID, job.Sequence, job.Token))
}
func (s *Store) FinishSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time, diagnostic string) error {
// Never trust an optimistic worker return as proof of a full export.
result, err := s.db.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE run_id=? AND sequence=? AND status='running' AND token=? AND `+completeExportJobEvidence, job.RunID, job.Sequence, job.Token)
if err != nil {
return err
}
if n, _ := result.RowsAffected(); n == 1 {
return nil
}
status := "pending"
delay := 20 * time.Second
if job.Attempts == 2 {
delay = 60 * time.Second
}
if job.Attempts >= SessionExportJobAttempts {
status = "exhausted"
}
nextAt := now.Add(delay).UnixMilli()
if status == "exhausted" {
nextAt = 0
}
if diagnostic == "" {
diagnostic = "Native session evidence remains incomplete; upstream recovery is not guaranteed"
}
if len(diagnostic) > 2048 {
diagnostic = diagnostic[:2048]
}
return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status=?,token='',lease_until=0,next_at=?,diagnostic=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, status, nextAt, diagnostic, job.RunID, job.Sequence, job.Token))
}
func (s *Store) SessionExportJob(ctx context.Context, id int64, sequence int) (*SessionExportJob, error) {
var job SessionExportJob
err := s.db.QueryRowContext(ctx, `SELECT run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("session export recovery: %w", err)
}
return &job, nil
}