187 lines
9.6 KiB
Go
187 lines
9.6 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func (s *Store) initRuns() error {
|
|
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS pipeline_runs (
|
|
id INTEGER PRIMARY KEY, pipeline_id INTEGER NOT NULL, version INTEGER NOT NULL,
|
|
request_key TEXT NOT NULL UNIQUE, workspace TEXT NOT NULL, allow_tools INTEGER NOT NULL,
|
|
title TEXT NOT NULL DEFAULT '', title_source TEXT NOT NULL DEFAULT '', title_model TEXT NOT NULL DEFAULT '',
|
|
model TEXT NOT NULL DEFAULT '', stage_models TEXT NOT NULL DEFAULT '{}',
|
|
write_paths TEXT NOT NULL, input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}',
|
|
status TEXT NOT NULL DEFAULT 'queued', outcome TEXT NOT NULL DEFAULT '', current_stage TEXT NOT NULL DEFAULT '',
|
|
diagnostic TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT,
|
|
FOREIGN KEY(pipeline_id,version) REFERENCES pipeline_versions(pipeline_id,version));
|
|
CREATE TABLE IF NOT EXISTS run_steps (
|
|
run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), sequence INTEGER NOT NULL, stage_key TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'running', outcome TEXT NOT NULL DEFAULT '', target TEXT NOT NULL DEFAULT '',
|
|
input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}', diagnostic TEXT NOT NULL DEFAULT '',
|
|
provider TEXT NOT NULL DEFAULT '', model TEXT NOT NULL DEFAULT '',
|
|
session_id TEXT NOT NULL DEFAULT '', agent_active INTEGER NOT NULL DEFAULT 0, transcript TEXT NOT NULL DEFAULT '',
|
|
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT, PRIMARY KEY(run_id,sequence));
|
|
CREATE INDEX IF NOT EXISTS run_latest ON pipeline_runs(id DESC);
|
|
CREATE INDEX IF NOT EXISTS run_steps_session ON run_steps(session_id);
|
|
CREATE TABLE IF NOT EXISTS step_notifications (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, receipts_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_file_changes (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, changes_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_control (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, evidence_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_artifacts (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, artifacts_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_verdicts (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, verdict_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_session_exports (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_usage (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, usage_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS run_session_owners (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS run_retries (
|
|
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
|
|
CREATE TABLE IF NOT EXISTS retry_checkpoints (
|
|
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
|
|
mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL,
|
|
source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
|
|
CREATE TABLE IF NOT EXISTS resource_owners (
|
|
resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
|
|
stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, pool_name TEXT NOT NULL DEFAULT '',
|
|
cleanup_confirmed INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '',
|
|
acquired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
|
|
CREATE INDEX IF NOT EXISTS resource_owner_run ON resource_owners(run_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS resource_owner_visit ON resource_owners(run_id);
|
|
CREATE TABLE IF NOT EXISTS resource_waiters (
|
|
queue_order INTEGER PRIMARY KEY AUTOINCREMENT, resource_key TEXT NOT NULL,
|
|
run_id INTEGER NOT NULL UNIQUE REFERENCES pipeline_runs(id), stage_key TEXT NOT NULL,
|
|
sequence INTEGER NOT NULL, token TEXT NOT NULL, queued_at TEXT NOT NULL);
|
|
CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,queue_order);`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_pools (
|
|
name TEXT PRIMARY KEY, members_json TEXT NOT NULL, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);`); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("resource_waiters", "pool_name", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("resource_owners", "pool_name", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
|
return err
|
|
}
|
|
if err := s.migrateRunTitles(); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_recovery_audit (
|
|
resource_key TEXT NOT NULL, run_id INTEGER NOT NULL, stage_key TEXT NOT NULL,
|
|
sequence INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL,
|
|
evidence TEXT NOT NULL, result TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)`); err != nil {
|
|
return err
|
|
}
|
|
return s.migrateRunModels()
|
|
}
|
|
|
|
// migrateRunModels adds the launch-time model-override columns to a database
|
|
// created before per-run overrides existed. Each column is added only when
|
|
// missing, so an upgrade is idempotent across repeated Open calls and existing
|
|
// rows keep their defaults (no model override, no recorded step model).
|
|
func (s *Store) migrateRunModels() error {
|
|
for _, column := range [][3]string{
|
|
{"pipeline_runs", "model", "TEXT NOT NULL DEFAULT ''"},
|
|
{"pipeline_runs", "stage_models", "TEXT NOT NULL DEFAULT '{}'"},
|
|
{"run_steps", "provider", "TEXT NOT NULL DEFAULT ''"},
|
|
{"run_steps", "model", "TEXT NOT NULL DEFAULT ''"},
|
|
} {
|
|
if err := s.addColumnIfMissing(column[0], column[1], column[2]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// migrateRunTitles adds the title columns to a database created before run
|
|
// titles existed. Adding a column with a default leaves every existing row valid
|
|
// and untouched, so old runs still list.
|
|
func (s *Store) migrateRunTitles() error {
|
|
for _, column := range [][2]string{
|
|
{"title", "TEXT NOT NULL DEFAULT ''"},
|
|
{"title_source", "TEXT NOT NULL DEFAULT ''"},
|
|
{"title_model", "TEXT NOT NULL DEFAULT ''"},
|
|
} {
|
|
if err := s.addColumnIfMissing("pipeline_runs", column[0], column[1]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addColumnIfMissing adds a column only when the table predates it, making an
|
|
// upgrade idempotent across repeated Open calls without rewriting history.
|
|
func (s *Store) addColumnIfMissing(table, column, definition string) error {
|
|
rows, err := s.db.Query("PRAGMA table_info(" + table + ")")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var cid, notnull, pk int
|
|
var name, ctype string
|
|
var dflt sql.NullString
|
|
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
|
return err
|
|
}
|
|
if name == column {
|
|
return nil
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
_, err = s.db.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " " + definition)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) RecoverRuns(ctx context.Context) error {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
// No replay: pending visits never began. Uncertain owners survive recovery,
|
|
// including the crash window before BeginStep/session registration.
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped while queued for a resource; pending visit was not resumed',status='interrupted',finished_at=CURRENT_TIMESTAMP,resource_wait='' WHERE id IN (SELECT run_id FROM resource_waiters); DELETE FROM resource_waiters"); err != nil {
|
|
return err
|
|
}
|
|
if err := recoverResourceOwners(ctx, tx); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=CASE WHEN EXISTS(SELECT 1 FROM resource_owners o WHERE o.run_id=run_steps.run_id AND o.sequence=run_steps.sequence) THEN agent_active ELSE 0 END,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>?", protocol.StepStatusWaiting); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic='Server stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>?", protocol.RunStatusWaiting); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET diagnostic='Server stopped; resource ownership quarantined: inspect retained session, tool termination and restoration before reconciliation' WHERE id IN (SELECT run_id FROM resource_owners)"); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|