Files
pipelines/internal/pipeline/runs_mutate.go
T
alan d35739cb49 Release session export gates whose recovery is exhausted or overdue
An exhausted export recovery job left its run parked forever in the
export wait, reserving the workspace and refusing later scheduled runs
(runs 221/226: OpenCode sessions above the 8 MiB export bound are
always truncated, so every retry stays partial).

A gate is now releasable when its job is exhausted: the visit keeps its
partial/unavailable evidence and diagnostic (still repairable), the run
continues to its next stage or terminal, and the workspace is freed.
Gates older than 15 minutes with a pending, lease-expired or missing job
are exhausted by a bounded wait. The worker reconciles on its first
tick after startup and every 30 seconds, so existing wedged runs recover
after a restart. Truncation diagnostics now name the size bound.
2026-11-01 17:24:56 +00:00

412 lines
18 KiB
Go

package pipeline
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func (s *Store) SaveNotifications(ctx context.Context, id int64, sequence int, receipts []protocol.NotificationReceipt) error {
data, err := json.Marshal(receipts)
if err != nil {
return err
}
_, err = s.db.ExecContext(ctx, `INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET receipts_json=excluded.receipts_json`, id, sequence, string(data))
return err
}
func affected(result sql.Result, err error) error {
if err != nil {
return err
}
n, err := result.RowsAffected()
if err != nil {
return err
}
if n != 1 {
return fmt.Errorf("invalid run state transition")
}
return nil
}
// writeStepEvidence stores one visit's evidence row, replacing the same visit's
// previous record. The runner persists a visit more than once: an export gate
// re-saves it after the session export adds files, and a settlement may follow a
// cleanup save. Those later writes describe the same visit more completely, and
// failing them aborted whole runs on a raw primary-key error. Per run the writes
// are serialized, so the last record is the newest known evidence.
func writeStepEvidence(ctx context.Context, tx *sql.Tx, table, column string, id int64, sequence int, data string) error {
_, err := tx.ExecContext(ctx,
"INSERT INTO "+table+"(run_id,sequence,"+column+") VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET "+column+"=excluded."+column,
id, sequence, data)
return err
}
func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key string, input map[string]any) error {
encoded, err := json.Marshal(input)
if err != nil {
return err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'') NOT IN ('complete','exhausted'))`, key, id)); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
return err
}
// Consuming a recovered handoff and starting its successor are atomic. A
// crash before this transaction leaves the gate available after restart.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND sequence<?`, id, sequence); err != nil {
return err
}
return tx.Commit()
}
// stageArtifactsEvidence is the persisted per-step artifact list plus its
// bounded-scan counters; it is kept in a separate table like the other
// step-scoped evidence so run_steps needs no schema migration on old databases.
type stageArtifactsEvidence struct {
Files []protocol.StageArtifact `json:"files"`
Skipped int `json:"skipped,omitempty"`
Truncated int `json:"truncated,omitempty"`
}
// stageVerdictEvidence is the generated one-line verdict reason plus its
// provenance. Like the other step-scoped evidence it lives in its own table, so
// run_steps needs no schema migration and an old database gains the empty table
// on Open with no backfill. Only a model-generated line is ever stored; the
// read-time fallback is always derived, so existing runs still show a verdict.
type stageVerdictEvidence struct {
Reason string `json:"reason"`
Source string `json:"source"`
}
func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep) error {
return s.finishStep(ctx, id, step, false)
}
// FinishStepExportBlocked retains the computed step and its routing target, but
// checkpoints the handoff separately: the output is not yet eligible for reuse.
func (s *Store) FinishStepExportBlocked(ctx context.Context, id int64, step protocol.RunStep) error {
return s.finishStep(ctx, id, step, true)
}
func (s *Store) finishStep(ctx context.Context, id int64, step protocol.RunStep, exportBlocked bool) error {
output, err := json.Marshal(step.Output)
if err != nil {
return err
}
// Bound the stored transcript without corrupting structured JSON. A large
// transcript becomes a valid bounded preview envelope preserving stage_limits
// and failure/format diagnostics; the exact Output is saved separately below.
step.Transcript = stage.BoundTranscript(step.Transcript, step.SessionID)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
var requested bool
if err := tx.QueryRowContext(ctx, "SELECT cancel_requested FROM pipeline_runs WHERE id=?", id).Scan(&requested); err != nil {
return err
}
if requested && step.Status != "running" {
step.Status, step.Outcome, step.Target = "canceled", "", ""
step.Diagnostic = "Canceled by operator; execution returned"
}
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CASE WHEN ?='running' THEN NULL ELSE CURRENT_TIMESTAMP END
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, step.Status, id, step.Sequence)); err != nil {
return err
}
if step.FileChanges != nil {
data, err := json.Marshal(step.FileChanges)
if err != nil {
return err
}
if err = writeStepEvidence(ctx, tx, "step_file_changes", "changes_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
if step.Control != nil {
data, err := json.Marshal(step.Control)
if err != nil {
return err
}
if err := writeStepEvidence(ctx, tx, "step_control", "evidence_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
if len(step.Artifacts) > 0 || step.ArtifactsSkipped > 0 || step.ArtifactsTruncated > 0 {
data, err := json.Marshal(stageArtifactsEvidence{Files: step.Artifacts, Skipped: step.ArtifactsSkipped, Truncated: step.ArtifactsTruncated})
if err != nil {
return err
}
if err := writeStepEvidence(ctx, tx, "step_artifacts", "artifacts_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
// Persist only a generated model line and its provenance; the read-time
// fallback is always derived in GetRun, so old runs stay intact.
if step.Verdict != nil && strings.TrimSpace(step.Verdict.Reason) != "" {
data, err := json.Marshal(stageVerdictEvidence{Reason: step.Verdict.Reason, Source: step.Verdict.ReasonSource})
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO step_verdicts(run_id,sequence,verdict_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET verdict_json=excluded.verdict_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
if step.SessionExport != nil {
data, err := json.Marshal(step.SessionExport)
if err != nil {
return err
}
if err := writeStepEvidence(ctx, tx, "step_session_exports", "export_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
if err := enqueueStepExportJob(ctx, tx, id, step.Sequence); err != nil {
return err
}
if exportBlocked && !requested {
if _, err := tx.ExecContext(ctx, `INSERT INTO session_export_gates(run_id,sequence) VALUES(?,?)`, id, step.Sequence); err != nil {
return err
}
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',diagnostic='Session export pending; computed visit retained',output_json=?,finished_at=NULL WHERE id=? AND status='running'`, string(output), id)); err != nil {
return err
}
}
// The reported usage the stage already measured for budget accounting is
// persisted as step-scoped evidence in the same transaction, mirroring the
// other step evidence. It is written only when the stage actually reported
// usage, so a historical or non-model step stays absent rather than zero.
if step.Usage != nil {
data, err := json.Marshal(step.Usage)
if err != nil {
return err
}
if err := writeStepEvidence(ctx, tx, "step_usage", "usage_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
return tx.Commit()
}
// MarkWaiting persists a human stage's pending state in one transaction: the
// step becomes waiting with self-contained evidence (its operator instructions
// and ordered options) and the run reports the same key it is waiting on. It
// leaves finished_at NULL for both and adds no goroutine, timer or new column.
func (s *Store) MarkWaiting(ctx context.Context, id int64, sequence int, key string, evidence map[string]any) error {
data, err := json.Marshal(evidence)
if err != nil {
return err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,output_json=? WHERE run_id=? AND sequence=? AND status='running'`,
protocol.StepStatusWaiting, string(data), id, sequence)); err != nil {
return err
}
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status=?,current_stage=? WHERE id=? AND status IN ('queued','running')`,
protocol.RunStatusWaiting, key, id)); err != nil {
return err
}
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
return err
}
return tx.Commit()
}
// ResolveDecision records an operator's choice for a waiting human stage and
// routes the run along that option's edge, all in one short transaction. It
// reads the pinned graph and the pending step from persisted rows, so it is
// correct after a restart and needs no live goroutine. Validation is idempotent:
// a repeated decision for an already-decided visit with the same option returns
// applied=false and no error, an unknown option is refused with the valid list,
// and a stage that is not waiting is refused with its current state.
func (s *Store) ResolveDecision(ctx context.Context, id int64, stageKey, option, feedback string) (protocol.RunStep, bool, error) {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return protocol.RunStep{}, false, err
}
defer tx.Rollback()
var exportPending bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&exportPending); err != nil {
return protocol.RunStep{}, false, err
}
if exportPending {
return protocol.RunStep{}, false, fmt.Errorf("run %d is awaiting session export, not a human decision", id)
}
var status, currentStage, graphJSON string
if err := tx.QueryRowContext(ctx, `SELECT r.status,r.current_stage,v.graph_json FROM pipeline_runs r
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&status, &currentStage, &graphJSON); err != nil {
if err == sql.ErrNoRows {
return protocol.RunStep{}, false, fmt.Errorf("run not found")
}
return protocol.RunStep{}, false, err
}
if stageKey == "" {
stageKey = currentStage
// Prefer the actual waiting step's stage; current_stage is normally the
// waiting stage, but this keeps the decision correct if it ever drifts.
var waitingKey string
if err := tx.QueryRowContext(ctx, `SELECT stage_key FROM run_steps WHERE run_id=? AND status=? ORDER BY sequence DESC LIMIT 1`, id, protocol.StepStatusWaiting).Scan(&waitingKey); err == nil {
stageKey = waitingKey
} else if err != sql.ErrNoRows {
return protocol.RunStep{}, false, err
}
}
var graph protocol.Graph
if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
return protocol.RunStep{}, false, err
}
var edges map[string]string
found := false
for _, declared := range graph.Stages {
if declared.Key == stageKey {
if stage.EffectiveType(declared.Type) != stage.HumanType {
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not a human decision stage", stageKey)
}
edges = declared.Edges
found = true
break
}
}
if !found {
return protocol.RunStep{}, false, fmt.Errorf("stage %q is not part of this run", stageKey)
}
options := stage.HumanOptions(edges)
valid := false
for _, candidate := range options {
if candidate == option {
valid = true
break
}
}
if !valid {
return protocol.RunStep{}, false, fmt.Errorf("unknown option %q for stage %s; valid options: %s", option, stageKey, strings.Join(options, ", "))
}
var sequence int
var stepStatus, stepOutcome string
err = tx.QueryRowContext(ctx, `SELECT sequence,status,outcome FROM run_steps WHERE run_id=? AND stage_key=? ORDER BY sequence DESC LIMIT 1`, id, stageKey).Scan(&sequence, &stepStatus, &stepOutcome)
if err == sql.ErrNoRows {
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not waiting", stageKey)
}
if err != nil {
return protocol.RunStep{}, false, err
}
if stepStatus != protocol.StepStatusWaiting {
if stepStatus == "completed" && stepOutcome == option {
return protocol.RunStep{Sequence: sequence, Stage: stageKey, Status: "completed", Outcome: stepOutcome, Target: edges[stepOutcome]}, false, nil
}
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not waiting (state %s)", stageKey, stepStatus)
}
if status != protocol.RunStatusWaiting && status != protocol.RunStatusRunning {
return protocol.RunStep{}, false, fmt.Errorf("run %d is not waiting (state %s)", id, status)
}
target := edges[option]
// The operator's choice and feedback are the step's evidence. The feedback
// stays in the output only - never the step diagnostic - so a deliberate,
// successful decision is not presented as a stop or error; the derived
// verdict reads the feedback as the step's reason.
output := map[string]any{protocol.DecisionOptionKey: option, protocol.DecisionFeedbackKey: feedback}
encoded, err := json.Marshal(output)
if err != nil {
return protocol.RunStep{}, false, err
}
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status='completed',outcome=?,target=?,output_json=?,finished_at=CURRENT_TIMESTAMP
WHERE run_id=? AND sequence=? AND status=?`, option, target, string(encoded), id, sequence, protocol.StepStatusWaiting)); err != nil {
return protocol.RunStep{}, false, err
}
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running' WHERE id=? AND status IN (?,?)`, id, protocol.RunStatusWaiting, protocol.RunStatusRunning)); err != nil {
return protocol.RunStep{}, false, err
}
if err := tx.Commit(); err != nil {
return protocol.RunStep{}, false, err
}
return protocol.RunStep{Sequence: sequence, Stage: stageKey, Status: "completed", Outcome: option, Target: target, Output: output}, true, nil
}
// CancelWaiting cancels a run that is waiting on a person without a live
// goroutine. It is a single transaction that marks the run canceled and its
// waiting step canceled; a run that is not waiting is reported, not silently
// ignored.
func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,
diagnostic=CASE WHEN EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id) THEN 'Canceled while awaiting session export; computed visit retained' ELSE 'Canceled while waiting for a decision' END,
finished_at=CURRENT_TIMESTAMP WHERE id=? AND (status=? OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id, protocol.RunStatusWaiting)
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return err
}
if n != 1 {
return fmt.Errorf("run is not active or waiting")
}
if _, err := tx.ExecContext(ctx, `UPDATE run_steps SET status='canceled',diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE run_id=? AND status=?`, id, protocol.StepStatusWaiting); err != nil {
return err
}
return tx.Commit()
}
// Cancel records intent only; it is not evidence that the executor stopped.
func (s *Store) Cancel(ctx context.Context, id int64) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cancel_requested=1,
diagnostic=CASE WHEN cleanup_unconfirmed=1 THEN diagnostic ELSE 'Stop requested; waiting for execution and cleanup to stop' END
WHERE id=? AND (status IN ('queued','running','waiting') OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id))
}
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
data, err := json.Marshal(output)
if err != nil {
return err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET
status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END,
outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)); err != nil {
return err
}
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
return err
}
// Terminal targets have no successor BeginStep to consume their gate. An
// exhausted job is terminal too: its incomplete evidence stays recorded.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status IN ('complete','exhausted'))`, id); err != nil {
return err
}
return tx.Commit()
}
func (s *Store) MarkCleanupUnconfirmed(ctx context.Context, id int64) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=1,
diagnostic='Agent cleanup was not confirmed; workspace remains reserved until successful restart recovery'
WHERE id=? AND status IN ('queued','running')`, id))
}