Files
pipelines/internal/pipeline/runs_schema.go
T

218 lines
10 KiB
Go

package pipeline
import (
"context"
"database/sql"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func (s *Store) initRuns() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS pipeline_runs (
id INTEGER PRIMARY KEY, pipeline_id INTEGER NOT NULL, version INTEGER NOT NULL,
request_key TEXT NOT NULL UNIQUE, workspace TEXT NOT NULL, allow_tools INTEGER NOT NULL,
title TEXT NOT NULL DEFAULT '', title_source TEXT NOT NULL DEFAULT '', title_model TEXT NOT NULL DEFAULT '',
model TEXT NOT NULL DEFAULT '', stage_models TEXT NOT NULL DEFAULT '{}', notify_mode TEXT NOT NULL DEFAULT 'auto',
write_paths TEXT NOT NULL, input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}',
status TEXT NOT NULL DEFAULT 'queued', outcome TEXT NOT NULL DEFAULT '', current_stage TEXT NOT NULL DEFAULT '',
diagnostic TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT,
FOREIGN KEY(pipeline_id,version) REFERENCES pipeline_versions(pipeline_id,version));
CREATE TABLE IF NOT EXISTS run_steps (
run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), sequence INTEGER NOT NULL, stage_key TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'running', outcome TEXT NOT NULL DEFAULT '', target TEXT NOT NULL DEFAULT '',
input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}', diagnostic TEXT NOT NULL DEFAULT '',
provider TEXT NOT NULL DEFAULT '', model TEXT NOT NULL DEFAULT '',
session_id TEXT NOT NULL DEFAULT '', agent_active INTEGER NOT NULL DEFAULT 0, transcript TEXT NOT NULL DEFAULT '',
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT, PRIMARY KEY(run_id,sequence));
CREATE INDEX IF NOT EXISTS run_latest ON pipeline_runs(id DESC);
CREATE INDEX IF NOT EXISTS run_steps_session ON run_steps(session_id);
CREATE TABLE IF NOT EXISTS step_notifications (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, receipts_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS step_file_changes (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, changes_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS step_control (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, evidence_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS step_artifacts (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, artifacts_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS step_verdicts (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, verdict_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS step_session_exports (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS step_usage (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, usage_json TEXT NOT NULL,
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS run_session_owners (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS run_retries (
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
CREATE TABLE IF NOT EXISTS retry_checkpoints (
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL,
source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);`)
if err != nil {
return err
}
if err := s.migrateRunTitles(); err != nil {
return err
}
for _, column := range []string{"cancel_requested", "cleanup_unconfirmed"} {
if err := s.addColumnIfMissing("pipeline_runs", column, "INTEGER NOT NULL DEFAULT 0"); err != nil {
return err
}
}
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_repairs (id INTEGER PRIMARY KEY, run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL, FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE); CREATE INDEX IF NOT EXISTS session_export_repairs_visit ON session_export_repairs(run_id,sequence,id);`); err != nil {
return err
}
if err := s.addColumnIfMissing("pipeline_runs", "output_format", "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
if err := s.addColumnIfMissing("pipeline_runs", "notify_mode", "TEXT NOT NULL DEFAULT 'auto'"); err != nil {
return err
}
if err := s.dropResourceSchema(); err != nil {
return err
}
if err := s.migrateRunModels(); err != nil {
return err
}
return s.initSessionExportJobs()
}
// dropResourceSchema removes the retired resource pool, ownership, wait queue
// and recovery tables, plus the queue snapshot column. It is idempotent: a
// database that predates the feature or was already migrated opens cleanly,
// while an older database loses only the removed feature's state.
func (s *Store) dropResourceSchema() error {
for _, table := range []string{"resource_owners", "resource_waiters", "resource_pools", "resource_recovery_audit"} {
if _, err := s.db.Exec("DROP TABLE IF EXISTS " + table); err != nil {
return err
}
}
return s.dropColumnIfMissing("pipeline_runs", "resource_wait")
}
// dropColumnIfMissing removes a retired column only when it is still present, so
// repeated Open calls and databases created after the removal stay idempotent.
func (s *Store) dropColumnIfMissing(table, column string) error {
rows, err := s.db.Query("PRAGMA table_info(" + table + ")")
if err != nil {
return err
}
found := false
for rows.Next() {
var cid, notnull, pk int
var name, ctype string
var dflt sql.NullString
if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk); err != nil {
rows.Close()
return err
}
if name == column {
found = true
}
}
if err := rows.Err(); err != nil {
rows.Close()
return err
}
rows.Close()
if !found {
return nil
}
_, err = s.db.Exec("ALTER TABLE " + table + " DROP COLUMN " + column)
return err
}
// migrateRunModels adds the launch-time model-override columns to a database
// created before per-run overrides existed. Each column is added only when
// missing, so an upgrade is idempotent across repeated Open calls and existing
// rows keep their defaults (no model override, no recorded step model).
func (s *Store) migrateRunModels() error {
for _, column := range [][3]string{
{"pipeline_runs", "model", "TEXT NOT NULL DEFAULT ''"},
{"pipeline_runs", "stage_models", "TEXT NOT NULL DEFAULT '{}'"},
{"run_steps", "provider", "TEXT NOT NULL DEFAULT ''"},
{"run_steps", "model", "TEXT NOT NULL DEFAULT ''"},
} {
if err := s.addColumnIfMissing(column[0], column[1], column[2]); err != nil {
return err
}
}
return nil
}
// migrateRunTitles adds the title columns to a database created before run
// titles existed. Adding a column with a default leaves every existing row valid
// and untouched, so old runs still list.
func (s *Store) migrateRunTitles() error {
for _, column := range [][2]string{
{"title", "TEXT NOT NULL DEFAULT ''"},
{"title_source", "TEXT NOT NULL DEFAULT ''"},
{"title_model", "TEXT NOT NULL DEFAULT ''"},
} {
if err := s.addColumnIfMissing("pipeline_runs", column[0], column[1]); err != nil {
return err
}
}
return nil
}
// addColumnIfMissing adds a column only when the table predates it, making an
// upgrade idempotent across repeated Open calls without rewriting history.
func (s *Store) addColumnIfMissing(table, column, definition string) error {
rows, err := s.db.Query("PRAGMA table_info(" + table + ")")
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var cid, notnull, pk int
var name, ctype string
var dflt sql.NullString
if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk); err != nil {
return err
}
if name == column {
return nil
}
}
if err := rows.Err(); err != nil {
return err
}
_, err = s.db.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " " + definition)
return err
}
func (s *Store) RecoverRuns(ctx context.Context) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
// No replay: pending visits never began. A cleanup agent that was still
// active is marked interrupted, and its run is interrupted below.
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)", protocol.StepStatusWaiting); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
return err
}
// An export checkpoint is a paused handoff, not an interrupted model call.
// Also migrate the earlier failed-gate representation before clients attach.
if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',finished_at=NULL,diagnostic='Session export pending; computed visit retained'
WHERE status IN ('queued','running','failed','interrupted','waiting') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
WHERE g.run_id=pipeline_runs.id AND s.status IN ('completed','failed') AND s.agent_active=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.sequence>g.sequence OR a.agent_active<>0)))`); err != nil {
return err
}
return tx.Commit()
}