214 lines
11 KiB
Go
214 lines
11 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
const SessionExportJobAttempts = 3
|
|
|
|
// SessionExportJob is an immutable ownership snapshot plus durable recovery
|
|
// state. Attempts count recovery operations, not the export CLI's inner reads.
|
|
type SessionExportJob struct {
|
|
RunID int64 `json:"run_id"`
|
|
Sequence int `json:"sequence"`
|
|
SessionID string `json:"session_id"`
|
|
Backend string `json:"backend"`
|
|
Directory string `json:"directory"`
|
|
Reference string `json:"-"`
|
|
Status string `json:"status"`
|
|
Attempts int `json:"attempts"`
|
|
NextAt int64 `json:"next_attempt_at_ms"`
|
|
Diagnostic string `json:"diagnostic,omitempty"`
|
|
Token string `json:"-"`
|
|
}
|
|
|
|
func (s *Store) initSessionExportJobs() error {
|
|
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_jobs (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL,
|
|
session_id TEXT NOT NULL, backend TEXT NOT NULL, directory TEXT NOT NULL, reference TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','complete','exhausted')), attempts INTEGER NOT NULL DEFAULT 0 CHECK(attempts BETWEEN 0 AND 3),
|
|
next_at INTEGER NOT NULL, lease_until INTEGER NOT NULL DEFAULT 0, token TEXT NOT NULL DEFAULT '', diagnostic TEXT NOT NULL DEFAULT '',
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE);
|
|
CREATE INDEX IF NOT EXISTS session_export_jobs_due ON session_export_jobs(status,next_at,lease_until);`)
|
|
if err == nil {
|
|
_, err = s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_gates (
|
|
run_id INTEGER PRIMARY KEY, sequence INTEGER NOT NULL,
|
|
FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE)`)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Missing or incomplete original evidence is recoverable only with an exact
|
|
// native ownership record. No backend guesses, prompt parsing or stage reruns.
|
|
const enqueueExportJobs = `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at)
|
|
SELECT s.run_id,s.sequence,s.session_id,o.backend,r.workspace,o.reference,?
|
|
FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id
|
|
JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence
|
|
LEFT JOIN step_session_exports x ON x.run_id=s.run_id AND x.sequence=s.sequence
|
|
WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND (o.backend IN ('v1','v2') OR (o.backend='pi' AND o.reference<>''))
|
|
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=s.run_id AND j.sequence=s.sequence)
|
|
AND NOT (COALESCE(json_extract(x.export_json,'$.status'),'complete')='complete'
|
|
AND COALESCE(json_extract(x.export_json,'$.session_id'),'')=s.session_id
|
|
AND COALESCE(json_extract(x.export_json,'$.backend'),'')=o.backend
|
|
AND COALESCE(json_extract(x.export_json,'$.messages'),0)>0
|
|
AND COALESCE(json_extract(x.export_json,'$.html'),'')<>''
|
|
AND COALESCE(json_extract(x.export_json,'$.html_bytes'),0)>0
|
|
AND COALESCE(json_extract(x.export_json,'$.truncated'),0)=0
|
|
AND COALESCE(json_extract(x.export_json,'$.diagnostic'),'')='')`
|
|
|
|
func enqueueStepExportJob(ctx context.Context, tx *sql.Tx, id int64, sequence int) error {
|
|
_, err := tx.ExecContext(ctx, enqueueExportJobs+` AND s.run_id=? AND s.sequence=? ON CONFLICT(run_id,sequence) DO NOTHING`, time.Now().Add(5*time.Second).UnixMilli(), id, sequence)
|
|
return err
|
|
}
|
|
|
|
// ReconcileSessionExportJobs recovers interrupted initial capture and historical
|
|
// missing jobs from source-of-truth step ownership. Existing schedules, exhausted
|
|
// budgets and completed jobs are never reset, even across repeated restarts.
|
|
func (s *Store) ReconcileSessionExportJobs(ctx context.Context, now time.Time) error {
|
|
_, err := s.db.ExecContext(ctx, enqueueExportJobs+` LIMIT 256 ON CONFLICT(run_id,sequence) DO NOTHING`, now.Add(5*time.Second).UnixMilli())
|
|
return err
|
|
}
|
|
|
|
const exactExportJobOwner = `EXISTS(SELECT 1 FROM run_steps s JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence JOIN pipeline_runs r ON r.id=s.run_id WHERE s.run_id=j.run_id AND s.sequence=j.sequence AND s.session_id=j.session_id AND o.backend=j.backend AND o.reference=j.reference AND r.workspace=j.directory)`
|
|
|
|
// A complete repair must match the job's identity and have nonempty verified
|
|
// file evidence. This also closes the crash window after repair persistence but
|
|
// before acknowledgement of its job lease, without writing another export.
|
|
const completeExportJobEvidence = `EXISTS(SELECT 1 FROM session_export_repairs p WHERE p.run_id=j.run_id AND p.sequence=j.sequence
|
|
AND json_extract(p.export_json,'$.status')='complete'
|
|
AND json_extract(p.export_json,'$.session_id')=j.session_id
|
|
AND json_extract(p.export_json,'$.backend')=j.backend
|
|
AND json_extract(p.export_json,'$.directory')=j.directory
|
|
AND json_extract(p.export_json,'$.messages')>0 AND json_extract(p.export_json,'$.html_bytes')>0
|
|
AND COALESCE(json_extract(p.export_json,'$.html'),'')<>''
|
|
AND COALESCE(json_extract(p.export_json,'$.truncated'),0)=0
|
|
AND COALESCE(json_extract(p.export_json,'$.diagnostic'),'')='')`
|
|
|
|
// cleanupGuard supports the older audit branch without editing its store or
|
|
// cancellation schema. On current main the persisted flag is mandatory in the
|
|
// claim predicate. The legacy schema has no cleanup-unconfirmed state.
|
|
func (s *Store) exportJobCleanupGuard(ctx context.Context) (string, error) {
|
|
rows, err := s.db.QueryContext(ctx, "PRAGMA table_info(pipeline_runs)")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var cid, notnull, pk int
|
|
var name, kind string
|
|
var def sql.NullString
|
|
if err := rows.Scan(&cid, &name, &kind, ¬null, &def, &pk); err != nil {
|
|
return "", err
|
|
}
|
|
if name == "cleanup_unconfirmed" {
|
|
return " AND r.cleanup_unconfirmed=0", nil
|
|
}
|
|
}
|
|
return "", rows.Err()
|
|
}
|
|
|
|
func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease time.Duration) (*SessionExportJob, error) {
|
|
if lease <= 0 {
|
|
return nil, fmt.Errorf("session export recovery lease must be positive")
|
|
}
|
|
guard, err := s.exportJobCleanupGuard(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
// A persisted successful repair wins, including after a process restart.
|
|
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE status<>'complete' AND `+completeExportJobEvidence); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Original session ownership changed or is unavailable; automatic recovery refused' WHERE status IN ('pending','running') AND NOT `+exactExportJobOwner); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Recovery budget exhausted; last worker stopped before acknowledgement' WHERE (status='pending' OR (status='running' AND lease_until<=?)) AND attempts>=?`, now.UnixMilli(), SessionExportJobAttempts); err != nil {
|
|
return nil, err
|
|
}
|
|
var job SessionExportJob
|
|
err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic
|
|
FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence
|
|
WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts<?
|
|
AND (r.status IN ('succeeded','failed','canceled','interrupted') OR (r.status='waiting' AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=j.run_id AND g.sequence=j.sequence))) AND s.status NOT IN ('running','waiting')`+guard+`
|
|
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=j.run_id OR a.session_id=j.session_id) AND a.agent_active<>0)
|
|
ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
|
|
if err == sql.ErrNoRows {
|
|
return nil, tx.Commit()
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var nonce [16]byte
|
|
if _, err := rand.Read(nonce[:]); err != nil {
|
|
return nil, err
|
|
}
|
|
job.Token = hex.EncodeToString(nonce[:])
|
|
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='running',attempts=attempts+1,token=?,lease_until=? WHERE run_id=? AND sequence=?`, job.Token, now.Add(lease).UnixMilli(), job.RunID, job.Sequence); err != nil {
|
|
return nil, err
|
|
}
|
|
job.Attempts++
|
|
job.Status = "running"
|
|
if err := tx.Commit(); err != nil {
|
|
return nil, err
|
|
}
|
|
return &job, nil
|
|
}
|
|
|
|
// DeferSessionExportJob returns a lease without spending a retry when the
|
|
// runner's in-memory cleanup gate is still held. No native operation ran.
|
|
func (s *Store) DeferSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time) error {
|
|
return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status='pending',attempts=attempts-1,token='',lease_until=0,next_at=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, now.Add(5*time.Second).UnixMilli(), job.RunID, job.Sequence, job.Token))
|
|
}
|
|
|
|
func (s *Store) FinishSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time, diagnostic string) error {
|
|
// Never trust an optimistic worker return as proof of a full export.
|
|
result, err := s.db.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE run_id=? AND sequence=? AND status='running' AND token=? AND `+completeExportJobEvidence, job.RunID, job.Sequence, job.Token)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := result.RowsAffected(); n == 1 {
|
|
return nil
|
|
}
|
|
status := "pending"
|
|
delay := 20 * time.Second
|
|
if job.Attempts == 2 {
|
|
delay = 60 * time.Second
|
|
}
|
|
if job.Attempts >= SessionExportJobAttempts {
|
|
status = "exhausted"
|
|
}
|
|
nextAt := now.Add(delay).UnixMilli()
|
|
if status == "exhausted" {
|
|
nextAt = 0
|
|
}
|
|
if diagnostic == "" {
|
|
diagnostic = "Native session evidence remains incomplete; upstream recovery is not guaranteed"
|
|
}
|
|
if len(diagnostic) > 2048 {
|
|
diagnostic = diagnostic[:2048]
|
|
}
|
|
return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status=?,token='',lease_until=0,next_at=?,diagnostic=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, status, nextAt, diagnostic, job.RunID, job.Sequence, job.Token))
|
|
}
|
|
|
|
func (s *Store) SessionExportJob(ctx context.Context, id int64, sequence int) (*SessionExportJob, error) {
|
|
var job SessionExportJob
|
|
err := s.db.QueryRowContext(ctx, `SELECT run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("session export recovery: %w", err)
|
|
}
|
|
return &job, nil
|
|
}
|