Files
pipelines/internal/pipeline/runs.go
T
alan 05ae530442 Make Pipelines Docker-only with OpenCode and Pi
Install current verified harness releases inside Docker, preserve persistent accounts and settings, and provide separate interactive and pipeline model defaults with terminal onboarding and fuzzy pickers.

Add Docker-aware authoring transport, Pi stage execution, bounded workflow controls, reliable final-result readback and usage settlement, and finite short-lived CLI pipe draining. Keep dated installation evidence and packaged authoring guidance consistent.

Verified with a real Docker installation, both harness stages and interactive authors, the built-in plan/implement/review starter, independent host race/vet checks, installer tests, and packaged skill tests. Generated release receipts stay ignored in clean checkouts.
2026-10-11 23:00:25 +01:00

420 lines
16 KiB
Go

package pipeline
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"fmt"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func (s *Store) initRuns() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS pipeline_runs (
id INTEGER PRIMARY KEY, pipeline_id INTEGER NOT NULL, version INTEGER NOT NULL,
request_key TEXT NOT NULL UNIQUE, workspace TEXT NOT NULL, allow_tools INTEGER NOT NULL,
write_paths TEXT NOT NULL, input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}',
status TEXT NOT NULL DEFAULT 'queued', outcome TEXT NOT NULL DEFAULT '', current_stage TEXT NOT NULL DEFAULT '',
diagnostic TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT,
FOREIGN KEY(pipeline_id,version) REFERENCES pipeline_versions(pipeline_id,version));
CREATE TABLE IF NOT EXISTS run_steps (
run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), sequence INTEGER NOT NULL, stage_key TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'running', outcome TEXT NOT NULL DEFAULT '', target TEXT NOT NULL DEFAULT '',
input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}', diagnostic TEXT NOT NULL DEFAULT '',
session_id TEXT NOT NULL DEFAULT '', agent_active INTEGER NOT NULL DEFAULT 0, transcript TEXT NOT NULL DEFAULT '',
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT, PRIMARY KEY(run_id,sequence));
CREATE INDEX IF NOT EXISTS run_latest ON pipeline_runs(id DESC);
CREATE TABLE IF NOT EXISTS step_notifications (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, receipts_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS step_file_changes (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, changes_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS step_control (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, evidence_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS run_session_owners (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS run_retries (
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));`)
return err
}
func (s *Store) CreateRun(ctx context.Context, d *protocol.PipelineDetail, key, workspace string, tools bool, paths []string, input map[string]any, retryOf ...int64) (int64, bool, error) {
var parent int64
if len(retryOf) > 0 {
parent = retryOf[0]
}
if parent < 0 {
return 0, false, fmt.Errorf("invalid retry parent")
}
encoded, err := json.Marshal(input)
if err != nil {
return 0, false, err
}
writes, err := json.Marshal(paths)
if err != nil {
return 0, false, err
}
s.mu.Lock()
defer s.mu.Unlock()
var id, pid int64
var version int
var oldInput, oldWorkspace, oldPaths string
var oldTools bool
err = s.db.QueryRowContext(ctx, "SELECT id,pipeline_id,version,input_json,workspace,write_paths,allow_tools FROM pipeline_runs WHERE request_key=?", key).Scan(&id, &pid, &version, &oldInput, &oldWorkspace, &oldPaths, &oldTools)
if err == nil {
var oldParent int64
if err := s.db.QueryRowContext(ctx, "SELECT COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=?),0)", id).Scan(&oldParent); err != nil {
return 0, false, err
}
if pid != d.ID || version != d.Version || oldInput != string(encoded) || oldWorkspace != workspace || oldPaths != string(writes) || oldTools != tools || oldParent != parent {
return 0, false, fmt.Errorf("request key already used for a different run")
}
return id, false, nil
}
if err != sql.ErrNoRows {
return 0, false, err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return 0, false, err
}
defer tx.Rollback()
result, err := tx.ExecContext(ctx, "INSERT INTO pipeline_runs(pipeline_id,version,request_key,workspace,allow_tools,write_paths,input_json) VALUES (?,?,?,?,?,?,?)", d.ID, d.Version, key, workspace, tools, string(writes), string(encoded))
if err != nil {
return 0, false, err
}
id, err = result.LastInsertId()
if err != nil {
return 0, false, err
}
if parent > 0 {
if _, err := tx.ExecContext(ctx, "INSERT INTO run_retries(run_id,original_run_id) VALUES(?,?)", id, parent); err != nil {
return 0, false, err
}
}
if err := tx.Commit(); err != nil {
return 0, false, err
}
return id, true, nil
}
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.finished_at),''),r.diagnostic,
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed')
FROM pipeline_runs r JOIN pipelines p ON p.id=r.pipeline_id JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps); err != nil {
return err
}
var values map[string]any
if err := decodeValues(output, &values); err != nil {
return err
}
r.ResultPreview = protocol.OutputPreview(values)
return nil
}
func (s *Store) ListRuns(ctx context.Context) ([]protocol.RunSummary, error) {
rows, err := s.db.QueryContext(ctx, runSelect+" ORDER BY r.id DESC")
if err != nil {
return nil, err
}
defer rows.Close()
items := []protocol.RunSummary{}
for rows.Next() {
var r protocol.RunSummary
if err := scanRun(rows, &r); err != nil {
return nil, err
}
items = append(items, r)
}
return items, rows.Err()
}
func (s *Store) RunByKey(ctx context.Context, key string) (int64, error) {
var id int64
err := s.db.QueryRowContext(ctx, "SELECT id FROM pipeline_runs WHERE request_key=?", key).Scan(&id)
if err == sql.ErrNoRows {
return 0, nil
}
return id, err
}
func decodeValues(text string, v *map[string]any) error {
d := json.NewDecoder(bytes.NewBufferString(text))
d.UseNumber()
return d.Decode(v)
}
func (s *Store) GetRun(ctx context.Context, id int64) (*protocol.RunDetail, error) {
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
r := &protocol.RunDetail{}
if err := scanRun(tx.QueryRowContext(ctx, runSelect+" WHERE r.id=?", id), &r.RunSummary); err != nil {
if err == sql.ErrNoRows {
return nil, fmt.Errorf("run not found")
}
return nil, err
}
var input, output, paths string
var graphJSON string
if err := tx.QueryRowContext(ctx, "SELECT graph_json FROM pipeline_versions WHERE pipeline_id=? AND version=?", r.PipelineID, r.Version).Scan(&graphJSON); err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(graphJSON), &r.Graph); err != nil {
return nil, err
}
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0) FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf); err != nil {
return nil, err
}
if err := decodeValues(input, &r.Input); err != nil {
return nil, err
}
if err := decodeValues(output, &r.Output); err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(paths), &r.WritePaths); err != nil {
return nil, err
}
rows, err := tx.QueryContext(ctx, `SELECT sequence,stage_key,status,outcome,target,input_json,output_json,diagnostic,session_id,agent_active,transcript,
COALESCE((SELECT receipts_json FROM step_notifications n WHERE n.run_id=run_steps.run_id AND n.sequence=run_steps.sequence),'[]'),
COALESCE((SELECT changes_json FROM step_file_changes f WHERE f.run_id=run_steps.run_id AND f.sequence=run_steps.sequence),'null'),
strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',finished_at),'') FROM run_steps WHERE run_id=? ORDER BY sequence`, id)
if err != nil {
return nil, err
}
r.Steps = []protocol.RunStep{}
for rows.Next() {
var step protocol.RunStep
var notifications string
var fileChanges string
if err := rows.Scan(&step.Sequence, &step.Stage, &step.Status, &step.Outcome, &step.Target, &input, &output, &step.Diagnostic, &step.SessionID, &step.AgentActive, &step.Transcript, &notifications, &fileChanges, &step.StartedAt, &step.FinishedAt); err != nil {
rows.Close()
return nil, err
}
if err := json.Unmarshal([]byte(notifications), &step.Notifications); err != nil {
rows.Close()
return nil, err
}
if err := json.Unmarshal([]byte(fileChanges), &step.FileChanges); err != nil {
rows.Close()
return nil, err
}
if err := decodeValues(input, &step.Input); err != nil {
rows.Close()
return nil, err
}
if err := decodeValues(output, &step.Output); err != nil {
rows.Close()
return nil, err
}
// Loaded below after the rows cursor closes to preserve SQLite connection use.
r.Steps = append(r.Steps, step)
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
for i := range r.Steps {
var data string
err := tx.QueryRowContext(ctx, "SELECT evidence_json FROM step_control WHERE run_id=? AND sequence=?", id, r.Steps[i].Sequence).Scan(&data)
if err == sql.ErrNoRows {
continue
}
if err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(data), &r.Steps[i].Control); err != nil {
return nil, err
}
}
if err := tx.Commit(); err != nil {
return nil, err
}
return r, nil
}
func (s *Store) SaveNotifications(ctx context.Context, id int64, sequence int, receipts []protocol.NotificationReceipt) error {
data, err := json.Marshal(receipts)
if err != nil {
return err
}
_, err = s.db.ExecContext(ctx, `INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET receipts_json=excluded.receipts_json`, id, sequence, string(data))
return err
}
func affected(result sql.Result, err error) error {
if err != nil {
return err
}
n, err := result.RowsAffected()
if err != nil {
return err
}
if n != 1 {
return fmt.Errorf("invalid run state transition")
}
return nil
}
func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key string, input map[string]any) error {
encoded, err := json.Marshal(input)
if err != nil {
return err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running')", key, id)); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
return err
}
return tx.Commit()
}
func (s *Store) RunSession(ctx context.Context, id int64, sequence int, session string, active bool, metadata ...string) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err = affected(tx.ExecContext(ctx, "UPDATE run_steps SET session_id=?,agent_active=? WHERE run_id=? AND sequence=? AND status='running'", session, active, id, sequence)); err != nil {
return err
}
if len(metadata) > 0 {
backend := metadata[0]
if backend == "" {
backend = "v2"
}
reference := ""
if len(metadata) > 1 {
reference = metadata[1]
}
if _, err = tx.ExecContext(ctx, "INSERT INTO run_session_owners(run_id,sequence,backend,reference) VALUES(?,?,?,?) ON CONFLICT(run_id,sequence) DO UPDATE SET backend=excluded.backend,reference=excluded.reference", id, sequence, backend, reference); err != nil {
return err
}
}
return tx.Commit()
}
type SessionOwner struct{ ID, Backend, Directory, Reference string }
func (s *Store) RunSessionOwner(ctx context.Context, id int64, sequence int) (SessionOwner, error) {
var owner SessionOwner
err := s.db.QueryRowContext(ctx, `SELECT s.session_id,COALESCE(o.backend,'v2'),r.workspace,COALESCE(o.reference,'') FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence WHERE s.run_id=? AND s.sequence=?`, id, sequence).Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference)
return owner, err
}
func (s *Store) ActiveSessionOwners(ctx context.Context) ([]SessionOwner, error) {
rows, err := s.db.QueryContext(ctx, `SELECT s.session_id,COALESCE(o.backend,'v2'),r.workspace,COALESCE(o.reference,'')
FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id LEFT JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence WHERE s.agent_active=1 AND s.session_id<>''`)
if err != nil {
return nil, err
}
defer rows.Close()
var owners []SessionOwner
for rows.Next() {
var owner SessionOwner
if err = rows.Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference); err != nil {
return nil, err
}
owners = append(owners, owner)
}
return owners, rows.Err()
}
func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep) error {
output, err := json.Marshal(step.Output)
if err != nil {
return err
}
// Bound the stored transcript without corrupting structured JSON. A large
// transcript becomes a valid bounded preview envelope preserving stage_limits
// and failure/format diagnostics; the exact Output is saved separately below.
step.Transcript = stage.BoundTranscript(step.Transcript, step.SessionID)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE run_steps SET status=?,outcome=?,target=?,output_json=?,diagnostic=?,transcript=?,finished_at=CURRENT_TIMESTAMP
WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, step.Target, string(output), step.Diagnostic, step.Transcript, id, step.Sequence)); err != nil {
return err
}
if step.FileChanges != nil {
data, err := json.Marshal(step.FileChanges)
if err != nil {
return err
}
if _, err = tx.ExecContext(ctx, "INSERT INTO step_file_changes(run_id,sequence,changes_json) VALUES(?,?,?)", id, step.Sequence, string(data)); err != nil {
return err
}
}
if step.Control != nil {
data, err := json.Marshal(step.Control)
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO step_control(run_id,sequence,evidence_json) VALUES(?,?,?)", id, step.Sequence, string(data)); err != nil {
return err
}
}
return tx.Commit()
}
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
data, err := json.Marshal(output)
if err != nil {
return err
}
return affected(s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status=?,outcome=?,diagnostic=?,output_json=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running')", status, outcome, diagnostic, string(data), id))
}
func (s *Store) ActiveAgentSessions(ctx context.Context) ([]string, error) {
rows, err := s.db.QueryContext(ctx, "SELECT DISTINCT session_id FROM run_steps WHERE agent_active=1 AND session_id<>''")
if err != nil {
return nil, err
}
defer rows.Close()
ids := []string{}
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, rows.Err()
}
func (s *Store) RecoverRuns(ctx context.Context) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status='running' OR agent_active=1"); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running')"); err != nil {
return err
}
return tx.Commit()
}