411 lines
18 KiB
Go
411 lines
18 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func (s *Store) SaveNotifications(ctx context.Context, id int64, sequence int, receipts []protocol.NotificationReceipt) error {
|
|
data, err := json.Marshal(receipts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = s.db.ExecContext(ctx, `INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET receipts_json=excluded.receipts_json`, id, sequence, string(data))
|
|
return err
|
|
}
|
|
|
|
func affected(result sql.Result, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n != 1 {
|
|
return fmt.Errorf("invalid run state transition")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// writeStepEvidence stores one visit's evidence row, replacing the same visit's
|
|
// previous record. The runner persists a visit more than once: an export gate
|
|
// re-saves it after the session export adds files, and a settlement may follow a
|
|
// cleanup save. Those later writes describe the same visit more completely, and
|
|
// failing them aborted whole runs on a raw primary-key error. Per run the writes
|
|
// are serialized, so the last record is the newest known evidence.
|
|
func writeStepEvidence(ctx context.Context, tx *sql.Tx, table, column string, id int64, sequence int, data string) error {
|
|
_, err := tx.ExecContext(ctx,
|
|
"INSERT INTO "+table+"(run_id,sequence,"+column+") VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET "+column+"=excluded."+column,
|
|
id, sequence, data)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key string, input map[string]any) error {
|
|
encoded, err := json.Marshal(input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0
|
|
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'')<>'complete')`, key, id)); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
|
|
return err
|
|
}
|
|
// Consuming a recovered handoff and starting its successor are atomic. A
|
|
// crash before this transaction leaves the gate available after restart.
|
|
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND sequence<?`, id, sequence); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// stageArtifactsEvidence is the persisted per-step artifact list plus its
|
|
// bounded-scan counters; it is kept in a separate table like the other
|
|
// step-scoped evidence so run_steps needs no schema migration on old databases.
|
|
type stageArtifactsEvidence struct {
|
|
Files []protocol.StageArtifact `json:"files"`
|
|
Skipped int `json:"skipped,omitempty"`
|
|
Truncated int `json:"truncated,omitempty"`
|
|
}
|
|
|
|
// stageVerdictEvidence is the generated one-line verdict reason plus its
|
|
// provenance. Like the other step-scoped evidence it lives in its own table, so
|
|
// run_steps needs no schema migration and an old database gains the empty table
|
|
// on Open with no backfill. Only a model-generated line is ever stored; the
|
|
// read-time fallback is always derived, so existing runs still show a verdict.
|
|
type stageVerdictEvidence struct {
|
|
Reason string `json:"reason"`
|
|
Source string `json:"source"`
|
|
}
|
|
|
|
func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep) error {
|
|
return s.finishStep(ctx, id, step, false)
|
|
}
|
|
|
|
// FinishStepExportBlocked retains the computed step and its routing target, but
|
|
// checkpoints the handoff separately: the output is not yet eligible for reuse.
|
|
func (s *Store) FinishStepExportBlocked(ctx context.Context, id int64, step protocol.RunStep) error {
|
|
return s.finishStep(ctx, id, step, true)
|
|
}
|
|
|
|
func (s *Store) finishStep(ctx context.Context, id int64, step protocol.RunStep, exportBlocked bool) error {
|
|
output, err := json.Marshal(step.Output)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Bound the stored transcript without corrupting structured JSON. A large
|
|
// transcript becomes a valid bounded preview envelope preserving stage_limits
|
|
// and failure/format diagnostics; the exact Output is saved separately below.
|
|
step.Transcript = stage.BoundTranscript(step.Transcript, step.SessionID)
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var requested bool
|
|
if err := tx.QueryRowContext(ctx, "SELECT cancel_requested FROM pipeline_runs WHERE id=?", id).Scan(&requested); err != nil {
|
|
return err
|
|
}
|
|
if requested && step.Status != "running" {
|
|
step.Status, step.Outcome, step.Target = "canceled", "", ""
|
|
step.Diagnostic = "Canceled by operator; execution returned"
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CASE WHEN ?='running' THEN NULL ELSE CURRENT_TIMESTAMP END
|
|
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, step.Status, id, step.Sequence)); err != nil {
|
|
return err
|
|
}
|
|
if step.FileChanges != nil {
|
|
data, err := json.Marshal(step.FileChanges)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = writeStepEvidence(ctx, tx, "step_file_changes", "changes_json", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if step.Control != nil {
|
|
data, err := json.Marshal(step.Control)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeStepEvidence(ctx, tx, "step_control", "evidence_json", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if len(step.Artifacts) > 0 || step.ArtifactsSkipped > 0 || step.ArtifactsTruncated > 0 {
|
|
data, err := json.Marshal(stageArtifactsEvidence{Files: step.Artifacts, Skipped: step.ArtifactsSkipped, Truncated: step.ArtifactsTruncated})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeStepEvidence(ctx, tx, "step_artifacts", "artifacts_json", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// Persist only a generated model line and its provenance; the read-time
|
|
// fallback is always derived in GetRun, so old runs stay intact.
|
|
if step.Verdict != nil && strings.TrimSpace(step.Verdict.Reason) != "" {
|
|
data, err := json.Marshal(stageVerdictEvidence{Reason: step.Verdict.Reason, Source: step.Verdict.ReasonSource})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO step_verdicts(run_id,sequence,verdict_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET verdict_json=excluded.verdict_json", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if step.SessionExport != nil {
|
|
data, err := json.Marshal(step.SessionExport)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeStepEvidence(ctx, tx, "step_session_exports", "export_json", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := enqueueStepExportJob(ctx, tx, id, step.Sequence); err != nil {
|
|
return err
|
|
}
|
|
if exportBlocked && !requested {
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO session_export_gates(run_id,sequence) VALUES(?,?)`, id, step.Sequence); err != nil {
|
|
return err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',diagnostic='Session export pending; computed visit retained',output_json=?,finished_at=NULL WHERE id=? AND status='running'`, string(output), id)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// The reported usage the stage already measured for budget accounting is
|
|
// persisted as step-scoped evidence in the same transaction, mirroring the
|
|
// other step evidence. It is written only when the stage actually reported
|
|
// usage, so a historical or non-model step stays absent rather than zero.
|
|
if step.Usage != nil {
|
|
data, err := json.Marshal(step.Usage)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeStepEvidence(ctx, tx, "step_usage", "usage_json", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// MarkWaiting persists a human stage's pending state in one transaction: the
|
|
// step becomes waiting with self-contained evidence (its operator instructions
|
|
// and ordered options) and the run reports the same key it is waiting on. It
|
|
// leaves finished_at NULL for both and adds no goroutine, timer or new column.
|
|
func (s *Store) MarkWaiting(ctx context.Context, id int64, sequence int, key string, evidence map[string]any) error {
|
|
data, err := json.Marshal(evidence)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,output_json=? WHERE run_id=? AND sequence=? AND status='running'`,
|
|
protocol.StepStatusWaiting, string(data), id, sequence)); err != nil {
|
|
return err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status=?,current_stage=? WHERE id=? AND status IN ('queued','running')`,
|
|
protocol.RunStatusWaiting, key, id)); err != nil {
|
|
return err
|
|
}
|
|
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// ResolveDecision records an operator's choice for a waiting human stage and
|
|
// routes the run along that option's edge, all in one short transaction. It
|
|
// reads the pinned graph and the pending step from persisted rows, so it is
|
|
// correct after a restart and needs no live goroutine. Validation is idempotent:
|
|
// a repeated decision for an already-decided visit with the same option returns
|
|
// applied=false and no error, an unknown option is refused with the valid list,
|
|
// and a stage that is not waiting is refused with its current state.
|
|
func (s *Store) ResolveDecision(ctx context.Context, id int64, stageKey, option, feedback string) (protocol.RunStep, bool, error) {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
defer tx.Rollback()
|
|
var exportPending bool
|
|
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&exportPending); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if exportPending {
|
|
return protocol.RunStep{}, false, fmt.Errorf("run %d is awaiting session export, not a human decision", id)
|
|
}
|
|
var status, currentStage, graphJSON string
|
|
if err := tx.QueryRowContext(ctx, `SELECT r.status,r.current_stage,v.graph_json FROM pipeline_runs r
|
|
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&status, ¤tStage, &graphJSON); err != nil {
|
|
if err == sql.ErrNoRows {
|
|
return protocol.RunStep{}, false, fmt.Errorf("run not found")
|
|
}
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if stageKey == "" {
|
|
stageKey = currentStage
|
|
// Prefer the actual waiting step's stage; current_stage is normally the
|
|
// waiting stage, but this keeps the decision correct if it ever drifts.
|
|
var waitingKey string
|
|
if err := tx.QueryRowContext(ctx, `SELECT stage_key FROM run_steps WHERE run_id=? AND status=? ORDER BY sequence DESC LIMIT 1`, id, protocol.StepStatusWaiting).Scan(&waitingKey); err == nil {
|
|
stageKey = waitingKey
|
|
} else if err != sql.ErrNoRows {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
}
|
|
var graph protocol.Graph
|
|
if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
var edges map[string]string
|
|
found := false
|
|
for _, declared := range graph.Stages {
|
|
if declared.Key == stageKey {
|
|
if stage.EffectiveType(declared.Type) != stage.HumanType {
|
|
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not a human decision stage", stageKey)
|
|
}
|
|
edges = declared.Edges
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return protocol.RunStep{}, false, fmt.Errorf("stage %q is not part of this run", stageKey)
|
|
}
|
|
options := stage.HumanOptions(edges)
|
|
valid := false
|
|
for _, candidate := range options {
|
|
if candidate == option {
|
|
valid = true
|
|
break
|
|
}
|
|
}
|
|
if !valid {
|
|
return protocol.RunStep{}, false, fmt.Errorf("unknown option %q for stage %s; valid options: %s", option, stageKey, strings.Join(options, ", "))
|
|
}
|
|
var sequence int
|
|
var stepStatus, stepOutcome string
|
|
err = tx.QueryRowContext(ctx, `SELECT sequence,status,outcome FROM run_steps WHERE run_id=? AND stage_key=? ORDER BY sequence DESC LIMIT 1`, id, stageKey).Scan(&sequence, &stepStatus, &stepOutcome)
|
|
if err == sql.ErrNoRows {
|
|
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not waiting", stageKey)
|
|
}
|
|
if err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if stepStatus != protocol.StepStatusWaiting {
|
|
if stepStatus == "completed" && stepOutcome == option {
|
|
return protocol.RunStep{Sequence: sequence, Stage: stageKey, Status: "completed", Outcome: stepOutcome, Target: edges[stepOutcome]}, false, nil
|
|
}
|
|
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not waiting (state %s)", stageKey, stepStatus)
|
|
}
|
|
if status != protocol.RunStatusWaiting && status != protocol.RunStatusRunning {
|
|
return protocol.RunStep{}, false, fmt.Errorf("run %d is not waiting (state %s)", id, status)
|
|
}
|
|
target := edges[option]
|
|
// The operator's choice and feedback are the step's evidence. The feedback
|
|
// stays in the output only - never the step diagnostic - so a deliberate,
|
|
// successful decision is not presented as a stop or error; the derived
|
|
// verdict reads the feedback as the step's reason.
|
|
output := map[string]any{protocol.DecisionOptionKey: option, protocol.DecisionFeedbackKey: feedback}
|
|
encoded, err := json.Marshal(output)
|
|
if err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status='completed',outcome=?,target=?,output_json=?,finished_at=CURRENT_TIMESTAMP
|
|
WHERE run_id=? AND sequence=? AND status=?`, option, target, string(encoded), id, sequence, protocol.StepStatusWaiting)); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running' WHERE id=? AND status IN (?,?)`, id, protocol.RunStatusWaiting, protocol.RunStatusRunning)); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
return protocol.RunStep{Sequence: sequence, Stage: stageKey, Status: "completed", Outcome: option, Target: target, Output: output}, true, nil
|
|
}
|
|
|
|
// CancelWaiting cancels a run that is waiting on a person without a live
|
|
// goroutine. It is a single transaction that marks the run canceled and its
|
|
// waiting step canceled; a run that is not waiting is reported, not silently
|
|
// ignored.
|
|
func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,
|
|
diagnostic=CASE WHEN EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id) THEN 'Canceled while awaiting session export; computed visit retained' ELSE 'Canceled while waiting for a decision' END,
|
|
finished_at=CURRENT_TIMESTAMP WHERE id=? AND (status=? OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id, protocol.RunStatusWaiting)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := res.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n != 1 {
|
|
return fmt.Errorf("run is not active or waiting")
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `UPDATE run_steps SET status='canceled',diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE run_id=? AND status=?`, id, protocol.StepStatusWaiting); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// Cancel records intent only; it is not evidence that the executor stopped.
|
|
func (s *Store) Cancel(ctx context.Context, id int64) error {
|
|
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cancel_requested=1,
|
|
diagnostic=CASE WHEN cleanup_unconfirmed=1 THEN diagnostic ELSE 'Stop requested; waiting for execution and cleanup to stop' END
|
|
WHERE id=? AND (status IN ('queued','running','waiting') OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id))
|
|
}
|
|
|
|
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
|
|
data, err := json.Marshal(output)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET
|
|
status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE ? END,
|
|
outcome=CASE WHEN cancel_requested=1 THEN '' ELSE ? END,
|
|
diagnostic=CASE WHEN cancel_requested=1 THEN 'Canceled by operator; execution stopped' ELSE ? END,
|
|
output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cleanup_unconfirmed=0
|
|
AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_active=1)`, status, outcome, diagnostic, string(data), id)); err != nil {
|
|
return err
|
|
}
|
|
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
|
|
return err
|
|
}
|
|
// Terminal targets have no successor BeginStep to consume their gate.
|
|
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status='complete')`, id); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
func (s *Store) MarkCleanupUnconfirmed(ctx context.Context, id int64) error {
|
|
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cleanup_unconfirmed=1,
|
|
diagnostic='Agent cleanup was not confirmed; workspace remains reserved until successful restart recovery'
|
|
WHERE id=? AND status IN ('queued','running')`, id))
|
|
}
|