Files
pipelines/internal/pipeline/session_export_gates.go
alan d35739cb49 Release session export gates whose recovery is exhausted or overdue
An exhausted export recovery job left its run parked forever in the
export wait, reserving the workspace and refusing later scheduled runs
(runs 221/226: OpenCode sessions above the 8 MiB export bound are
always truncated, so every retry stays partial).

A gate is now releasable when its job is exhausted: the visit keeps its
partial/unavailable evidence and diagnostic (still repairable), the run
continues to its next stage or terminal, and the workspace is freed.
Gates older than 15 minutes with a pending, lease-expired or missing job
are exhausted by a bounded wait. The worker reconciles on its first
tick after startup and every 30 seconds, so existing wedged runs recover
after a restart. Truncation diagnostics now name the size bound.
2026-11-01 17:24:56 +00:00

165 lines
8.1 KiB
Go

package pipeline
import (
"context"
"encoding/json"
"fmt"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func (s *Store) SessionExportBlocked(ctx context.Context, id int64) (bool, error) {
var blocked bool
err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&blocked)
return blocked, err
}
// PendingSessionExportGate authorizes only export repair of the exact completed
// visit, never a model call or a human decision. No live executor is implied by
// the workspace reservation maintained for this checkpoint.
func (s *Store) PendingSessionExportGate(ctx context.Context, id int64, sequence int) (bool, error) {
var pending bool
err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates g
JOIN pipeline_runs r ON r.id=g.run_id JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
WHERE g.run_id=? AND g.sequence=? AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND s.status NOT IN ('running','waiting')
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=g.run_id OR a.session_id=s.session_id) AND a.agent_active<>0))`, id, sequence).Scan(&pending)
return pending, err
}
// SessionExportReservations survive process restarts and cannot be confused
// with agent-active/executing flags in the UI.
func (s *Store) SessionExportReservations(ctx context.Context) (map[int64]string, error) {
rows, err := s.db.QueryContext(ctx, `SELECT r.id,r.workspace FROM session_export_gates g JOIN pipeline_runs r ON r.id=g.run_id WHERE r.cancel_requested=0 AND r.status<>'canceled'`)
if err != nil {
return nil, err
}
defer rows.Close()
reservations := map[int64]string{}
for rows.Next() {
var id int64
var workspace string
if err := rows.Scan(&id, &workspace); err != nil {
return nil, err
}
reservations[id] = workspace
}
return reservations, rows.Err()
}
// ReadySessionExportGates includes interrupted continuations. The gate remains
// durable until a successor is begun (or a terminal target is committed).
func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, error) {
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
JOIN pipeline_runs r ON r.id=g.run_id
WHERE r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+releasableGateJob+` LIMIT 64`)
if err != nil {
return nil, err
}
defer rows.Close()
ready := map[int64]int{}
for rows.Next() {
var id int64
var sequence int
if err := rows.Scan(&id, &sequence); err != nil {
return nil, err
}
ready[id] = sequence
}
return ready, rows.Err()
}
// ResumeSessionExportGate is a CAS, not a retry of the model invocation. A
// gate whose recovery job is exhausted is released exactly like a completed
// one: the visit keeps its partial/unavailable export evidence and the job's
// diagnostic, and the operator can still repair it later. When no export
// evidence was recorded at all, one unavailable record carries the diagnostic.
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND `+releasableGateJob+`)
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence)); err != nil {
return err
}
var job SessionExportJob
if err := tx.QueryRowContext(ctx, `SELECT session_id,backend,directory,status,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.SessionID, &job.Backend, &job.Directory, &job.Status, &job.Diagnostic); err != nil {
return err
}
if job.Status == "exhausted" {
var recorded bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM step_session_exports WHERE run_id=? AND sequence=?) OR EXISTS(SELECT 1 FROM session_export_repairs WHERE run_id=? AND sequence=?)`, id, sequence, id, sequence).Scan(&recorded); err != nil {
return err
}
if !recorded {
diagnostic := job.Diagnostic
if diagnostic == "" {
diagnostic = "Automatic session export recovery exhausted"
}
body, err := json.Marshal(protocol.SessionExport{Status: "unavailable", Source: "unavailable", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339), Diagnostic: diagnostic})
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,?,?)", id, sequence, string(body)); err != nil {
return err
}
}
}
return tx.Commit()
}
// releasableGateJob is a gate's terminal recovery state. A complete job must
// still match the exact original owner. An exhausted job is terminal whatever
// the reason (including changed ownership): waiting longer cannot help, so the
// handoff proceeds with the incomplete evidence instead of wedging the run and
// its workspace forever.
const releasableGateJob = `(j.status='exhausted' OR (j.status='complete' AND ` + exactExportJobOwner + `))`
// SessionExportGateMaxWait bounds how long a computed visit may wait for its
// export. The normal budget (three attempts with 45-second leases and up to a
// minute of backoff) finishes in a few minutes; this only catches a job that
// can never be claimed or acknowledged.
const SessionExportGateMaxWait = 15 * time.Minute
// ExpireSessionExportGates exhausts the recovery job of every gate older than
// maxWait whose job is still pending, or running past its lease, and gives a
// gate with no recovery job at all an exhausted one. A live lease is never
// stolen; it expires within its own bound and is exhausted on the next pass.
func (s *Store) ExpireSessionExportGates(ctx context.Context, now time.Time, maxWait time.Duration) (int64, error) {
cutoff := now.Add(-maxWait).UTC().Format(time.DateTime)
diagnostic := fmt.Sprintf("Session export wait exceeded %s; automatic recovery stopped", maxWait)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return 0, err
}
defer tx.Rollback()
result, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,
diagnostic=substr(?||CASE WHEN j.diagnostic<>'' THEN ' (last: '||j.diagnostic||')' ELSE '' END,1,2048)
WHERE (j.status='pending' OR (j.status='running' AND j.lease_until<=?))
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
WHERE g.run_id=j.run_id AND g.sequence=j.sequence AND s.finished_at IS NOT NULL AND s.finished_at<=?)`, diagnostic, now.UnixMilli(), cutoff)
if err != nil {
return 0, err
}
expired, _ := result.RowsAffected()
result, err = tx.ExecContext(ctx, `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic)
SELECT g.run_id,g.sequence,s.session_id,COALESCE(o.backend,''),r.workspace,COALESCE(o.reference,''),'exhausted',0,0,?
FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence JOIN pipeline_runs r ON r.id=g.run_id
LEFT JOIN run_session_owners o ON o.run_id=g.run_id AND o.sequence=g.sequence
WHERE s.finished_at IS NOT NULL AND s.finished_at<=?
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=g.run_id AND j.sequence=g.sequence)
ON CONFLICT(run_id,sequence) DO NOTHING`, "No automatic session export recovery exists for this visit; "+diagnostic, cutoff)
if err != nil {
return 0, err
}
created, _ := result.RowsAffected()
return expired + created, tx.Commit()
}