218 lines
10 KiB
Go
218 lines
10 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func (s *Store) initRuns() error {
|
|
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS pipeline_runs (
|
|
id INTEGER PRIMARY KEY, pipeline_id INTEGER NOT NULL, version INTEGER NOT NULL,
|
|
request_key TEXT NOT NULL UNIQUE, workspace TEXT NOT NULL, allow_tools INTEGER NOT NULL,
|
|
title TEXT NOT NULL DEFAULT '', title_source TEXT NOT NULL DEFAULT '', title_model TEXT NOT NULL DEFAULT '',
|
|
model TEXT NOT NULL DEFAULT '', stage_models TEXT NOT NULL DEFAULT '{}', notify_mode TEXT NOT NULL DEFAULT 'auto',
|
|
write_paths TEXT NOT NULL, input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}',
|
|
status TEXT NOT NULL DEFAULT 'queued', outcome TEXT NOT NULL DEFAULT '', current_stage TEXT NOT NULL DEFAULT '',
|
|
diagnostic TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT,
|
|
FOREIGN KEY(pipeline_id,version) REFERENCES pipeline_versions(pipeline_id,version));
|
|
CREATE TABLE IF NOT EXISTS run_steps (
|
|
run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), sequence INTEGER NOT NULL, stage_key TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'running', outcome TEXT NOT NULL DEFAULT '', target TEXT NOT NULL DEFAULT '',
|
|
input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}', diagnostic TEXT NOT NULL DEFAULT '',
|
|
provider TEXT NOT NULL DEFAULT '', model TEXT NOT NULL DEFAULT '',
|
|
session_id TEXT NOT NULL DEFAULT '', agent_active INTEGER NOT NULL DEFAULT 0, transcript TEXT NOT NULL DEFAULT '',
|
|
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT, PRIMARY KEY(run_id,sequence));
|
|
CREATE INDEX IF NOT EXISTS run_latest ON pipeline_runs(id DESC);
|
|
CREATE INDEX IF NOT EXISTS run_steps_session ON run_steps(session_id);
|
|
CREATE TABLE IF NOT EXISTS step_notifications (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, receipts_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_file_changes (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, changes_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_control (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, evidence_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_artifacts (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, artifacts_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_verdicts (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, verdict_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_session_exports (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_usage (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, usage_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS run_session_owners (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS run_retries (
|
|
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
|
|
CREATE TABLE IF NOT EXISTS retry_checkpoints (
|
|
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
|
|
mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL,
|
|
source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := s.migrateRunTitles(); err != nil {
|
|
return err
|
|
}
|
|
for _, column := range []string{"cancel_requested", "cleanup_unconfirmed"} {
|
|
if err := s.addColumnIfMissing("pipeline_runs", column, "INTEGER NOT NULL DEFAULT 0"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_repairs (id INTEGER PRIMARY KEY, run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL, FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE); CREATE INDEX IF NOT EXISTS session_export_repairs_visit ON session_export_repairs(run_id,sequence,id);`); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("pipeline_runs", "output_format", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("pipeline_runs", "notify_mode", "TEXT NOT NULL DEFAULT 'auto'"); err != nil {
|
|
return err
|
|
}
|
|
if err := s.dropResourceSchema(); err != nil {
|
|
return err
|
|
}
|
|
if err := s.migrateRunModels(); err != nil {
|
|
return err
|
|
}
|
|
return s.initSessionExportJobs()
|
|
}
|
|
|
|
// dropResourceSchema removes the retired resource pool, ownership, wait queue
|
|
// and recovery tables, plus the queue snapshot column. It is idempotent: a
|
|
// database that predates the feature or was already migrated opens cleanly,
|
|
// while an older database loses only the removed feature's state.
|
|
func (s *Store) dropResourceSchema() error {
|
|
for _, table := range []string{"resource_owners", "resource_waiters", "resource_pools", "resource_recovery_audit"} {
|
|
if _, err := s.db.Exec("DROP TABLE IF EXISTS " + table); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return s.dropColumnIfMissing("pipeline_runs", "resource_wait")
|
|
}
|
|
|
|
// dropColumnIfMissing removes a retired column only when it is still present, so
|
|
// repeated Open calls and databases created after the removal stay idempotent.
|
|
func (s *Store) dropColumnIfMissing(table, column string) error {
|
|
rows, err := s.db.Query("PRAGMA table_info(" + table + ")")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
found := false
|
|
for rows.Next() {
|
|
var cid, notnull, pk int
|
|
var name, ctype string
|
|
var dflt sql.NullString
|
|
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
if name == column {
|
|
found = true
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
rows.Close()
|
|
if !found {
|
|
return nil
|
|
}
|
|
_, err = s.db.Exec("ALTER TABLE " + table + " DROP COLUMN " + column)
|
|
return err
|
|
}
|
|
|
|
// migrateRunModels adds the launch-time model-override columns to a database
|
|
// created before per-run overrides existed. Each column is added only when
|
|
// missing, so an upgrade is idempotent across repeated Open calls and existing
|
|
// rows keep their defaults (no model override, no recorded step model).
|
|
func (s *Store) migrateRunModels() error {
|
|
for _, column := range [][3]string{
|
|
{"pipeline_runs", "model", "TEXT NOT NULL DEFAULT ''"},
|
|
{"pipeline_runs", "stage_models", "TEXT NOT NULL DEFAULT '{}'"},
|
|
{"run_steps", "provider", "TEXT NOT NULL DEFAULT ''"},
|
|
{"run_steps", "model", "TEXT NOT NULL DEFAULT ''"},
|
|
} {
|
|
if err := s.addColumnIfMissing(column[0], column[1], column[2]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// migrateRunTitles adds the title columns to a database created before run
|
|
// titles existed. Adding a column with a default leaves every existing row valid
|
|
// and untouched, so old runs still list.
|
|
func (s *Store) migrateRunTitles() error {
|
|
for _, column := range [][2]string{
|
|
{"title", "TEXT NOT NULL DEFAULT ''"},
|
|
{"title_source", "TEXT NOT NULL DEFAULT ''"},
|
|
{"title_model", "TEXT NOT NULL DEFAULT ''"},
|
|
} {
|
|
if err := s.addColumnIfMissing("pipeline_runs", column[0], column[1]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addColumnIfMissing adds a column only when the table predates it, making an
|
|
// upgrade idempotent across repeated Open calls without rewriting history.
|
|
func (s *Store) addColumnIfMissing(table, column, definition string) error {
|
|
rows, err := s.db.Query("PRAGMA table_info(" + table + ")")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var cid, notnull, pk int
|
|
var name, ctype string
|
|
var dflt sql.NullString
|
|
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
|
return err
|
|
}
|
|
if name == column {
|
|
return nil
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
_, err = s.db.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " " + definition)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) RecoverRuns(ctx context.Context) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
// No replay: pending visits never began. A cleanup agent that was still
|
|
// active is marked interrupted, and its run is interrupted below.
|
|
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)", protocol.StepStatusWaiting); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
|
|
return err
|
|
}
|
|
// An export checkpoint is a paused handoff, not an interrupted model call.
|
|
// Also migrate the earlier failed-gate representation before clients attach.
|
|
if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',finished_at=NULL,diagnostic='Session export pending; computed visit retained'
|
|
WHERE status IN ('queued','running','failed','interrupted','waiting') AND cancel_requested=0 AND cleanup_unconfirmed=0
|
|
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
|
|
WHERE g.run_id=pipeline_runs.id AND s.status IN ('completed','failed') AND s.agent_active=0
|
|
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.sequence>g.sequence OR a.agent_active<>0)))`); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|