d35739cb49
An exhausted export recovery job left its run parked forever in the export wait, reserving the workspace and refusing later scheduled runs (runs 221/226: OpenCode sessions above the 8 MiB export bound are always truncated, so every retry stays partial). A gate is now releasable when its job is exhausted: the visit keeps its partial/unavailable evidence and diagnostic (still repairable), the run continues to its next stage or terminal, and the workspace is freed. Gates older than 15 minutes with a pending, lease-expired or missing job are exhausted by a bounded wait. The worker reconciles on its first tick after startup and every 30 seconds, so existing wedged runs recover after a restart. Truncation diagnostics now name the size bound.
165 lines
8.1 KiB
Go
165 lines
8.1 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func (s *Store) SessionExportBlocked(ctx context.Context, id int64) (bool, error) {
|
|
var blocked bool
|
|
err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&blocked)
|
|
return blocked, err
|
|
}
|
|
|
|
// PendingSessionExportGate authorizes only export repair of the exact completed
|
|
// visit, never a model call or a human decision. No live executor is implied by
|
|
// the workspace reservation maintained for this checkpoint.
|
|
func (s *Store) PendingSessionExportGate(ctx context.Context, id int64, sequence int) (bool, error) {
|
|
var pending bool
|
|
err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates g
|
|
JOIN pipeline_runs r ON r.id=g.run_id JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
|
|
WHERE g.run_id=? AND g.sequence=? AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
|
|
AND s.status NOT IN ('running','waiting')
|
|
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=g.run_id OR a.session_id=s.session_id) AND a.agent_active<>0))`, id, sequence).Scan(&pending)
|
|
return pending, err
|
|
}
|
|
|
|
// SessionExportReservations survive process restarts and cannot be confused
|
|
// with agent-active/executing flags in the UI.
|
|
func (s *Store) SessionExportReservations(ctx context.Context) (map[int64]string, error) {
|
|
rows, err := s.db.QueryContext(ctx, `SELECT r.id,r.workspace FROM session_export_gates g JOIN pipeline_runs r ON r.id=g.run_id WHERE r.cancel_requested=0 AND r.status<>'canceled'`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
reservations := map[int64]string{}
|
|
for rows.Next() {
|
|
var id int64
|
|
var workspace string
|
|
if err := rows.Scan(&id, &workspace); err != nil {
|
|
return nil, err
|
|
}
|
|
reservations[id] = workspace
|
|
}
|
|
return reservations, rows.Err()
|
|
}
|
|
|
|
// ReadySessionExportGates includes interrupted continuations. The gate remains
|
|
// durable until a successor is begun (or a terminal target is committed).
|
|
func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, error) {
|
|
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
|
|
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
|
|
JOIN pipeline_runs r ON r.id=g.run_id
|
|
WHERE r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
|
|
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+releasableGateJob+` LIMIT 64`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
ready := map[int64]int{}
|
|
for rows.Next() {
|
|
var id int64
|
|
var sequence int
|
|
if err := rows.Scan(&id, &sequence); err != nil {
|
|
return nil, err
|
|
}
|
|
ready[id] = sequence
|
|
}
|
|
return ready, rows.Err()
|
|
}
|
|
|
|
// ResumeSessionExportGate is a CAS, not a retry of the model invocation. A
|
|
// gate whose recovery job is exhausted is released exactly like a completed
|
|
// one: the visit keeps its partial/unavailable export evidence and the job's
|
|
// diagnostic, and the operator can still repair it later. When no export
|
|
// evidence was recorded at all, one unavailable record carries the diagnostic.
|
|
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
|
|
WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
|
|
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND `+releasableGateJob+`)
|
|
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence)); err != nil {
|
|
return err
|
|
}
|
|
var job SessionExportJob
|
|
if err := tx.QueryRowContext(ctx, `SELECT session_id,backend,directory,status,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.SessionID, &job.Backend, &job.Directory, &job.Status, &job.Diagnostic); err != nil {
|
|
return err
|
|
}
|
|
if job.Status == "exhausted" {
|
|
var recorded bool
|
|
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM step_session_exports WHERE run_id=? AND sequence=?) OR EXISTS(SELECT 1 FROM session_export_repairs WHERE run_id=? AND sequence=?)`, id, sequence, id, sequence).Scan(&recorded); err != nil {
|
|
return err
|
|
}
|
|
if !recorded {
|
|
diagnostic := job.Diagnostic
|
|
if diagnostic == "" {
|
|
diagnostic = "Automatic session export recovery exhausted"
|
|
}
|
|
body, err := json.Marshal(protocol.SessionExport{Status: "unavailable", Source: "unavailable", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339), Diagnostic: diagnostic})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,?,?)", id, sequence, string(body)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// releasableGateJob is a gate's terminal recovery state. A complete job must
|
|
// still match the exact original owner. An exhausted job is terminal whatever
|
|
// the reason (including changed ownership): waiting longer cannot help, so the
|
|
// handoff proceeds with the incomplete evidence instead of wedging the run and
|
|
// its workspace forever.
|
|
const releasableGateJob = `(j.status='exhausted' OR (j.status='complete' AND ` + exactExportJobOwner + `))`
|
|
|
|
// SessionExportGateMaxWait bounds how long a computed visit may wait for its
|
|
// export. The normal budget (three attempts with 45-second leases and up to a
|
|
// minute of backoff) finishes in a few minutes; this only catches a job that
|
|
// can never be claimed or acknowledged.
|
|
const SessionExportGateMaxWait = 15 * time.Minute
|
|
|
|
// ExpireSessionExportGates exhausts the recovery job of every gate older than
|
|
// maxWait whose job is still pending, or running past its lease, and gives a
|
|
// gate with no recovery job at all an exhausted one. A live lease is never
|
|
// stolen; it expires within its own bound and is exhausted on the next pass.
|
|
func (s *Store) ExpireSessionExportGates(ctx context.Context, now time.Time, maxWait time.Duration) (int64, error) {
|
|
cutoff := now.Add(-maxWait).UTC().Format(time.DateTime)
|
|
diagnostic := fmt.Sprintf("Session export wait exceeded %s; automatic recovery stopped", maxWait)
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer tx.Rollback()
|
|
result, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,
|
|
diagnostic=substr(?||CASE WHEN j.diagnostic<>'' THEN ' (last: '||j.diagnostic||')' ELSE '' END,1,2048)
|
|
WHERE (j.status='pending' OR (j.status='running' AND j.lease_until<=?))
|
|
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
|
|
WHERE g.run_id=j.run_id AND g.sequence=j.sequence AND s.finished_at IS NOT NULL AND s.finished_at<=?)`, diagnostic, now.UnixMilli(), cutoff)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
expired, _ := result.RowsAffected()
|
|
result, err = tx.ExecContext(ctx, `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic)
|
|
SELECT g.run_id,g.sequence,s.session_id,COALESCE(o.backend,''),r.workspace,COALESCE(o.reference,''),'exhausted',0,0,?
|
|
FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence JOIN pipeline_runs r ON r.id=g.run_id
|
|
LEFT JOIN run_session_owners o ON o.run_id=g.run_id AND o.sequence=g.sequence
|
|
WHERE s.finished_at IS NOT NULL AND s.finished_at<=?
|
|
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=g.run_id AND j.sequence=g.sequence)
|
|
ON CONFLICT(run_id,sequence) DO NOTHING`, "No automatic session export recovery exists for this visit; "+diagnostic, cutoff)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
created, _ := result.RowsAffected()
|
|
return expired + created, tx.Commit()
|
|
}
|