9130ae8c7a
Add durable FIFO capacity-one resource leases for executable stages, including cleanup-confirmed release, conservative restart quarantine, queue state events, and CLI/TUI/Android waiting presentation. Add scheduler, cleanup, protocol, client, and Android coverage plus a proposed Android v3 resource draft. Checked packaged templates and authoring skill references: neither needs updating in this item. The per-item docs note records the shared documentation that docs-fold must update; no shared README or packaged reference was edited.
315 lines
13 KiB
Go
315 lines
13 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func (s *Store) SaveNotifications(ctx context.Context, id int64, sequence int, receipts []protocol.NotificationReceipt) error {
|
|
data, err := json.Marshal(receipts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = s.db.ExecContext(ctx, `INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET receipts_json=excluded.receipts_json`, id, sequence, string(data))
|
|
return err
|
|
}
|
|
|
|
func affected(result sql.Result, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n != 1 {
|
|
return fmt.Errorf("invalid run state transition")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key string, input map[string]any) error {
|
|
encoded, err := json.Marshal(input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=?,resource_wait='' WHERE id=? AND status IN ('queued','running')", key, id)); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// stageArtifactsEvidence is the persisted per-step artifact list plus its
|
|
// bounded-scan counters; it is kept in a separate table like the other
|
|
// step-scoped evidence so run_steps needs no schema migration on old databases.
|
|
type stageArtifactsEvidence struct {
|
|
Files []protocol.StageArtifact `json:"files"`
|
|
Skipped int `json:"skipped,omitempty"`
|
|
Truncated int `json:"truncated,omitempty"`
|
|
}
|
|
|
|
// stageVerdictEvidence is the generated one-line verdict reason plus its
|
|
// provenance. Like the other step-scoped evidence it lives in its own table, so
|
|
// run_steps needs no schema migration and an old database gains the empty table
|
|
// on Open with no backfill. Only a model-generated line is ever stored; the
|
|
// read-time fallback is always derived, so existing runs still show a verdict.
|
|
type stageVerdictEvidence struct {
|
|
Reason string `json:"reason"`
|
|
Source string `json:"source"`
|
|
}
|
|
|
|
func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep) error {
|
|
output, err := json.Marshal(step.Output)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Bound the stored transcript without corrupting structured JSON. A large
|
|
// transcript becomes a valid bounded preview envelope preserving stage_limits
|
|
// and failure/format diagnostics; the exact Output is saved separately below.
|
|
step.Transcript = stage.BoundTranscript(step.Transcript, step.SessionID)
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,provider=?,model=?,finished_at=CURRENT_TIMESTAMP
|
|
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, step.Provider, step.Model, id, step.Sequence)); err != nil {
|
|
return err
|
|
}
|
|
if step.FileChanges != nil {
|
|
data, err := json.Marshal(step.FileChanges)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err = tx.ExecContext(ctx, "INSERT INTO step_file_changes(run_id,sequence,changes_json) VALUES(?,?,?)", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if step.Control != nil {
|
|
data, err := json.Marshal(step.Control)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO step_control(run_id,sequence,evidence_json) VALUES(?,?,?)", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if len(step.Artifacts) > 0 || step.ArtifactsSkipped > 0 || step.ArtifactsTruncated > 0 {
|
|
data, err := json.Marshal(stageArtifactsEvidence{Files: step.Artifacts, Skipped: step.ArtifactsSkipped, Truncated: step.ArtifactsTruncated})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO step_artifacts(run_id,sequence,artifacts_json) VALUES(?,?,?)", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// Persist only a generated model line and its provenance; the read-time
|
|
// fallback is always derived in GetRun, so old runs stay intact.
|
|
if step.Verdict != nil && strings.TrimSpace(step.Verdict.Reason) != "" {
|
|
data, err := json.Marshal(stageVerdictEvidence{Reason: step.Verdict.Reason, Source: step.Verdict.ReasonSource})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO step_verdicts(run_id,sequence,verdict_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET verdict_json=excluded.verdict_json", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if step.SessionExport != nil {
|
|
data, err := json.Marshal(step.SessionExport)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO step_session_exports(run_id,sequence,export_json) VALUES(?,?,?)", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// The reported usage the stage already measured for budget accounting is
|
|
// persisted as step-scoped evidence in the same transaction, mirroring the
|
|
// other step evidence. It is written only when the stage actually reported
|
|
// usage, so a historical or non-model step stays absent rather than zero.
|
|
if step.Usage != nil {
|
|
data, err := json.Marshal(step.Usage)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,?,?)", id, step.Sequence, string(data)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// MarkWaiting persists a human stage's pending state in one transaction: the
|
|
// step becomes waiting with self-contained evidence (its operator instructions
|
|
// and ordered options) and the run reports the same key it is waiting on. It
|
|
// leaves finished_at NULL for both and adds no goroutine, timer or new column.
|
|
func (s *Store) MarkWaiting(ctx context.Context, id int64, sequence int, key string, evidence map[string]any) error {
|
|
data, err := json.Marshal(evidence)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,output_json=? WHERE run_id=? AND sequence=? AND status='running'`,
|
|
protocol.StepStatusWaiting, string(data), id, sequence)); err != nil {
|
|
return err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status=?,current_stage=? WHERE id=? AND status IN ('queued','running')`,
|
|
protocol.RunStatusWaiting, key, id)); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// ResolveDecision records an operator's choice for a waiting human stage and
|
|
// routes the run along that option's edge, all in one short transaction. It
|
|
// reads the pinned graph and the pending step from persisted rows, so it is
|
|
// correct after a restart and needs no live goroutine. Validation is idempotent:
|
|
// a repeated decision for an already-decided visit with the same option returns
|
|
// applied=false and no error, an unknown option is refused with the valid list,
|
|
// and a stage that is not waiting is refused with its current state.
|
|
func (s *Store) ResolveDecision(ctx context.Context, id int64, stageKey, option, feedback string) (protocol.RunStep, bool, error) {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
defer tx.Rollback()
|
|
var status, currentStage, graphJSON string
|
|
if err := tx.QueryRowContext(ctx, `SELECT r.status,r.current_stage,v.graph_json FROM pipeline_runs r
|
|
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&status, ¤tStage, &graphJSON); err != nil {
|
|
if err == sql.ErrNoRows {
|
|
return protocol.RunStep{}, false, fmt.Errorf("run not found")
|
|
}
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if stageKey == "" {
|
|
stageKey = currentStage
|
|
// Prefer the actual waiting step's stage; current_stage is normally the
|
|
// waiting stage, but this keeps the decision correct if it ever drifts.
|
|
var waitingKey string
|
|
if err := tx.QueryRowContext(ctx, `SELECT stage_key FROM run_steps WHERE run_id=? AND status=? ORDER BY sequence DESC LIMIT 1`, id, protocol.StepStatusWaiting).Scan(&waitingKey); err == nil {
|
|
stageKey = waitingKey
|
|
} else if err != sql.ErrNoRows {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
}
|
|
var graph protocol.Graph
|
|
if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
var edges map[string]string
|
|
found := false
|
|
for _, declared := range graph.Stages {
|
|
if declared.Key == stageKey {
|
|
edges = declared.Edges
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return protocol.RunStep{}, false, fmt.Errorf("stage %q is not part of this run", stageKey)
|
|
}
|
|
options := stage.HumanOptions(edges)
|
|
valid := false
|
|
for _, candidate := range options {
|
|
if candidate == option {
|
|
valid = true
|
|
break
|
|
}
|
|
}
|
|
if !valid {
|
|
return protocol.RunStep{}, false, fmt.Errorf("unknown option %q for stage %s; valid options: %s", option, stageKey, strings.Join(options, ", "))
|
|
}
|
|
var sequence int
|
|
var stepStatus, stepOutcome string
|
|
err = tx.QueryRowContext(ctx, `SELECT sequence,status,outcome FROM run_steps WHERE run_id=? AND stage_key=? ORDER BY sequence DESC LIMIT 1`, id, stageKey).Scan(&sequence, &stepStatus, &stepOutcome)
|
|
if err == sql.ErrNoRows {
|
|
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not waiting", stageKey)
|
|
}
|
|
if err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if stepStatus != protocol.StepStatusWaiting {
|
|
if stepStatus == "completed" && stepOutcome == option {
|
|
return protocol.RunStep{Sequence: sequence, Stage: stageKey, Status: "completed", Outcome: stepOutcome, Target: edges[stepOutcome]}, false, nil
|
|
}
|
|
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not waiting (state %s)", stageKey, stepStatus)
|
|
}
|
|
if status != protocol.RunStatusWaiting && status != protocol.RunStatusRunning {
|
|
return protocol.RunStep{}, false, fmt.Errorf("run %d is not waiting (state %s)", id, status)
|
|
}
|
|
target := edges[option]
|
|
// The operator's choice and feedback are the step's evidence. The feedback
|
|
// stays in the output only - never the step diagnostic - so a deliberate,
|
|
// successful decision is not presented as a stop or error; the derived
|
|
// verdict reads the feedback as the step's reason.
|
|
output := map[string]any{protocol.DecisionOptionKey: option, protocol.DecisionFeedbackKey: feedback}
|
|
encoded, err := json.Marshal(output)
|
|
if err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status='completed',outcome=?,target=?,output_json=?,finished_at=CURRENT_TIMESTAMP
|
|
WHERE run_id=? AND sequence=? AND status=?`, option, target, string(encoded), id, sequence, protocol.StepStatusWaiting)); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running' WHERE id=? AND status IN (?,?)`, id, protocol.RunStatusWaiting, protocol.RunStatusRunning)); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return protocol.RunStep{}, false, err
|
|
}
|
|
return protocol.RunStep{Sequence: sequence, Stage: stageKey, Status: "completed", Outcome: option, Target: target, Output: output}, true, nil
|
|
}
|
|
|
|
// CancelWaiting cancels a run that is waiting on a person without a live
|
|
// goroutine. It is a single transaction that marks the run canceled and its
|
|
// waiting step canceled; a run that is not waiting is reported, not silently
|
|
// ignored.
|
|
func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := res.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n != 1 {
|
|
return fmt.Errorf("run is not active or waiting")
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `UPDATE run_steps SET status='canceled',diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE run_id=? AND status=?`, id, protocol.StepStatusWaiting); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
|
|
data, err := json.Marshal(output)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
|
|
}
|