Files
pipelines/internal/pipeline/runs_mutate.go
T

411 lines
18 KiB
Go

package pipeline
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func (s *Store) SaveNotifications(ctx context.Context, id int64, sequence int, receipts []protocol.NotificationReceipt) error {
data, err := json.Marshal(receipts)
if err != nil {
return err
}
_, err = s.db.ExecContext(ctx, `INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET receipts_json=excluded.receipts_json`, id, sequence, string(data))
return err
}
func affected(result sql.Result, err error) error {
if err != nil {
return err
}
n, err := result.RowsAffected()
if err != nil {
return err
}
if n != 1 {
return fmt.Errorf("invalid run state transition")
}
return nil
}
// writeStepEvidence stores one visit's evidence row, replacing the same visit's
// previous record. The runner persists a visit more than once: an export gate
// re-saves it after the session export adds files, and a settlement may follow a
// cleanup save. Those later writes describe the same visit more completely, and
// failing them aborted whole runs on a raw primary-key error. Per run the writes
// are serialized, so the last record is the newest known evidence.
func writeStepEvidence(ctx context.Context, tx *sql.Tx, table, column string, id int64, sequence int, data string) error {
_, err := tx.ExecContext(ctx,
"INSERT INTO "+table+"(run_id,sequence,"+column+") VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET "+column+"=excluded."+column,
id, sequence, data)
return err
}
func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key string, input map[string]any) error {
encoded, err := json.Marshal(input)
if err != nil {
return err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'')<>'complete')`, key, id)); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
return err
}
// Consuming a recovered handoff and starting its successor are atomic. A
// crash before this transaction leaves the gate available after restart.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND sequence<?`, id, sequence); err != nil {
return err
}
return tx.Commit()
}
// stageArtifactsEvidence is the persisted per-step artifact list plus its
// bounded-scan counters; it is kept in a separate table like the other
// step-scoped evidence so run_steps needs no schema migration on old databases.
type stageArtifactsEvidence struct {
Files []protocol.StageArtifact `json:"files"`
Skipped int `json:"skipped,omitempty"`
Truncated int `json:"truncated,omitempty"`
}
// stageVerdictEvidence is the generated one-line verdict reason plus its
// provenance. Like the other step-scoped evidence it lives in its own table, so
// run_steps needs no schema migration and an old database gains the empty table
// on Open with no backfill. Only a model-generated line is ever stored; the
// read-time fallback is always derived, so existing runs still show a verdict.
type stageVerdictEvidence struct {
Reason string `json:"reason"`
Source string `json:"source"`
}
func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep) error {
return s.finishStep(ctx, id, step, false)
}
// FinishStepExportBlocked retains the computed step and its routing target, but
// checkpoints the handoff separately: the output is not yet eligible for reuse.
func (s *Store) FinishStepExportBlocked(ctx context.Context, id int64, step protocol.RunStep) error {
return s.finishStep(ctx, id, step, true)
}
func (s *Store) finishStep(ctx context.Context, id int64, step protocol.RunStep, exportBlocked bool) error {
output, err := json.Marshal(step.Output)
if err != nil {
return err
}
// Bound the stored transcript without corrupting structured JSON. A large
// transcript becomes a valid bounded preview envelope preserving stage_limits
// and failure/format diagnostics; the exact Output is saved separately below.
step.Transcript = stage.BoundTranscript(step.Transcript, step.SessionID)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
var requested bool
if err := tx.QueryRowContext(ctx, "SELECT cancel_requested FROM pipeline_runs WHERE id=?", id).Scan(&requested); err != nil {
return err
}
if requested && step.Status != "running" {
step.Status, step.Outcome, step.Target = "canceled", "", ""
step.Diagnostic = "Canceled by operator; execution returned"
}
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CASE WHEN ?='running' THEN NULL ELSE CURRENT_TIMESTAMP END
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, step.Status, id, step.Sequence)); err != nil {
return err
}
if step.FileChanges != nil {
data, err := json.Marshal(step.FileChanges)
if err != nil {
return err
}
if err = writeStepEvidence(ctx, tx, "step_file_changes", "changes_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
if step.Control != nil {
data, err := json.Marshal(step.Control)
if err != nil {
return err
}
if err := writeStepEvidence(ctx, tx, "step_control", "evidence_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
if len(step.Artifacts) > 0 || step.ArtifactsSkipped > 0 || step.ArtifactsTruncated > 0 {
data, err := json.Marshal(stageArtifactsEvidence{Files: step.Artifacts, Skipped: step.ArtifactsSkipped, Truncated: step.ArtifactsTruncated})
if err != nil {
return err
}
if err := writeStepEvidence(ctx, tx, "step_artifacts", "artifacts_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
// Persist only a generated model line and its provenance; the read-time
// fallback is always derived in GetRun, so old runs stay intact.
if step.Verdict != nil && strings.TrimSpace(step.Verdict.Reason) != "" {
data, err := json.Marshal(stageVerdictEvidence{Reason: step.Verdict.Reason, Source: step.Verdict.ReasonSource})
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO step_verdicts(run_id,sequence,verdict_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET verdict_json=excluded.verdict_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
if step.SessionExport != nil {
data, err := json.Marshal(step.SessionExport)
if err != nil {
return err
}
if err := writeStepEvidence(ctx, tx, "step_session_exports", "export_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
if err := enqueueStepExportJob(ctx, tx, id, step.Sequence); err != nil {
return err
}
if exportBlocked && !requested {
if _, err := tx.ExecContext(ctx, `INSERT INTO session_export_gates(run_id,sequence) VALUES(?,?)`, id, step.Sequence); err != nil {
return err
}
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',diagnostic='Session export pending; computed visit retained',output_json=?,finished_at=NULL WHERE id=? AND status='running'`, string(output), id)); err != nil {
return err
}
}
// The reported usage the stage already measured for budget accounting is
// persisted as step-scoped evidence in the same transaction, mirroring the
// other step evidence. It is written only when the stage actually reported
// usage, so a historical or non-model step stays absent rather than zero.
if step.Usage != nil {
data, err := json.Marshal(step.Usage)
if err != nil {
return err
}
if err := writeStepEvidence(ctx, tx, "step_usage", "usage_json", id, step.Sequence, string(data)); err != nil {
return err
}
}
return tx.Commit()
}
// MarkWaiting persists a human stage's pending state in one transaction: the
// step becomes waiting with self-contained evidence (its operator instructions
// and ordered options) and the run reports the same key it is waiting on. It
// leaves finished_at NULL for both and adds no goroutine, timer or new column.
func (s *Store) MarkWaiting(ctx context.Context, id int64, sequence int, key string, evidence map[string]any) error {
data, err := json.Marshal(evidence)
if err != nil {
return err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,output_json=? WHERE run_id=? AND sequence=? AND status='running'`,
protocol.StepStatusWaiting, string(data), id, sequence)); err != nil {
return err
}
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status=?,current_stage=? WHERE id=? AND status IN ('queued','running')`,
protocol.RunStatusWaiting, key, id)); err != nil {
return err
}
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
return err
}
return tx.Commit()
}
// ResolveDecision records an operator's choice for a waiting human stage and
// routes the run along that option's edge, all in one short transaction. It
// reads the pinned graph and the pending step from persisted rows, so it is
// correct after a restart and needs no live goroutine. Validation is idempotent:
// a repeated decision for an already-decided visit with the same option returns
// applied=false and no error, an unknown option is refused with the valid list,
// and a stage that is not waiting is refused with its current state.
func (s *Store) ResolveDecision(ctx context.Context, id int64, stageKey, option, feedback string) (protocol.RunStep, bool, error) {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return protocol.RunStep{}, false, err
}
defer tx.Rollback()
var exportPending bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&exportPending); err != nil {
return protocol.RunStep{}, false, err
}
if exportPending {
return protocol.RunStep{}, false, fmt.Errorf("run %d is awaiting session export, not a human decision", id)
}
var status, currentStage, graphJSON string
if err := tx.QueryRowContext(ctx, `SELECT r.status,r.current_stage,v.graph_json FROM pipeline_runs r
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&status, &currentStage, &graphJSON); err != nil {
if err == sql.ErrNoRows {
return protocol.RunStep{}, false, fmt.Errorf("run not found")
}
return protocol.RunStep{}, false, err
}
if stageKey == "" {
stageKey = currentStage
// Prefer the actual waiting step's stage; current_stage is normally the
// waiting stage, but this keeps the decision correct if it ever drifts.
var waitingKey string
if err := tx.QueryRowContext(ctx, `SELECT stage_key FROM run_steps WHERE run_id=? AND status=? ORDER BY sequence DESC LIMIT 1`, id, protocol.StepStatusWaiting).Scan(&waitingKey); err == nil {
stageKey = waitingKey
} else if err != sql.ErrNoRows {
return protocol.RunStep{}, false, err
}
}
var graph protocol.Graph
if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
return protocol.RunStep{}, false, err
}
var edges map[string]string
found := false
for _, declared := range graph.Stages {
if declared.Key == stageKey {
if stage.EffectiveType(declared.Type) != stage.HumanType {
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not a human decision stage", stageKey)
}
edges = declared.Edges
found = true
break
}
}
if !found {
return protocol.RunStep{}, false, fmt.Errorf("stage %q is not part of this run", stageKey)
}
options := stage.HumanOptions(edges)
valid := false
for _, candidate := range options {
if candidate == option {
valid = true
break
}
}
if !valid {
return protocol.RunStep{}, false, fmt.Errorf("unknown option %q for stage %s; valid options: %s", option, stageKey, strings.Join(options, ", "))
}
var sequence int
var stepStatus, stepOutcome string
err = tx.QueryRowContext(ctx, `SELECT sequence,status,outcome FROM run_steps WHERE run_id=? AND stage_key=? ORDER BY sequence DESC LIMIT 1`, id, stageKey).Scan(&sequence, &stepStatus, &stepOutcome)
if err == sql.ErrNoRows {
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not waiting", stageKey)
}
if err != nil {
return protocol.RunStep{}, false, err
}
if stepStatus != protocol.StepStatusWaiting {
if stepStatus == "completed" && stepOutcome == option {
return protocol.RunStep{Sequence: sequence, Stage: stageKey, Status: "completed", Outcome: stepOutcome, Target: edges[stepOutcome]}, false, nil
}
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not waiting (state %s)", stageKey, stepStatus)
}
if status != protocol.RunStatusWaiting && status != protocol.RunStatusRunning {
return protocol.RunStep{}, false, fmt.Errorf("run %d is not waiting (state %s)", id, status)
}
target := edges[option]
// The operator's choice and feedback are the step's evidence. The feedback
// stays in the output only - never the step diagnostic - so a deliberate,
// successful decision is not presented as a stop or error; the derived
// verdict reads the feedback as the step's reason.
output := map[string]any{protocol.DecisionOptionKey: option, protocol.DecisionFeedbackKey: feedback}
encoded, err := json.Marshal(output)
if err != nil {
return protocol.RunStep{}, false, err
}
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status='completed',outcome=?,target=?,output_json=?,finished_at=CURRENT_TIMESTAMP
WHERE run_id=? AND sequence=? AND status=?`, option, target, string(encoded), id, sequence, protocol.StepStatusWaiting)); err != nil {
return protocol.RunStep{}, false, err
}
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running' WHERE id=? AND status IN (?,?)`, id, protocol.RunStatusWaiting, protocol.RunStatusRunning)); err != nil {
return protocol.RunStep{}, false, err
}
if err := tx.Commit(); err != nil {
return protocol.RunStep{}, false, err
}
return protocol.RunStep{Sequence: sequence, Stage: stageKey, Status: "completed", Outcome: option, Target: target, Output: output}, true, nil
}
// CancelWaiting cancels a run that is waiting on a person without a live
// goroutine. It is a single transaction that marks the run canceled and its
// waiting step canceled; a run that is not waiting is reported, not silently
// ignored.
func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,
diagnostic=CASE WHEN EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id) THEN 'Canceled while awaiting session export; computed visit retained' ELSE 'Canceled while waiting for a decision' END,
finished_at=CURRENT_TIMESTAMP WHERE id=? AND (status=? OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id, protocol.RunStatusWaiting)
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return err
}
if n != 1 {
return fmt.Errorf("run is not active or waiting")
}
if _, err := tx.ExecContext(ctx, `UPDATE run_steps SET status='canceled',diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE run_id=? AND status=?`, id, protocol.StepStatusWaiting); err != nil {
return err
}
return tx.Commit()
}
// Cancel records intent only; it is not evidence that the executor stopped.
func (s *Store) Cancel(ctx context.Context, id int64) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cancel_requested=1,
diagnostic=CASE WHEN cleanup_unconfirmed=1 THEN diagnostic ELSE 'Stop requested; waiting for execution and cleanup to stop' END
WHERE id=? AND (status IN ('queued','running','waiting') OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id))
}
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
data, err := json.Marshal(output)
if err != nil {
return err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET
status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END,
outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)); err != nil {
return err
}
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
return err
}
// Terminal targets have no successor BeginStep to consume their gate.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status='complete')`, id); err != nil {
return err
}
return tx.Commit()
}
func (s *Store) MarkCleanupUnconfirmed(ctx context.Context, id int64) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=1,
diagnostic='Agent cleanup was not confirmed; workspace remains reserved until successful restart recovery'
WHERE id=? AND status IN ('queued','running')`, id))
}