a733a927f3
Preserve Settings as an independent navigation destination while landing the reviewed notification and skill-management updates. Keep the server and CLI skill contracts versioned and validated so the merged tree can be verified by the delivery gate.
265 lines
13 KiB
Go
265 lines
13 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func (s *Store) initRuns() error {
|
|
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS pipeline_runs (
|
|
id INTEGER PRIMARY KEY, pipeline_id INTEGER NOT NULL, version INTEGER NOT NULL,
|
|
request_key TEXT NOT NULL UNIQUE, workspace TEXT NOT NULL, allow_tools INTEGER NOT NULL,
|
|
title TEXT NOT NULL DEFAULT '', title_source TEXT NOT NULL DEFAULT '', title_model TEXT NOT NULL DEFAULT '',
|
|
model TEXT NOT NULL DEFAULT '', stage_models TEXT NOT NULL DEFAULT '{}', notify_mode TEXT NOT NULL DEFAULT 'auto',
|
|
write_paths TEXT NOT NULL, input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}',
|
|
status TEXT NOT NULL DEFAULT 'queued', outcome TEXT NOT NULL DEFAULT '', current_stage TEXT NOT NULL DEFAULT '',
|
|
diagnostic TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT,
|
|
FOREIGN KEY(pipeline_id,version) REFERENCES pipeline_versions(pipeline_id,version));
|
|
CREATE TABLE IF NOT EXISTS run_steps (
|
|
run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), sequence INTEGER NOT NULL, stage_key TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'running', outcome TEXT NOT NULL DEFAULT '', target TEXT NOT NULL DEFAULT '',
|
|
input_json TEXT NOT NULL, output_json TEXT NOT NULL DEFAULT '{}', diagnostic TEXT NOT NULL DEFAULT '',
|
|
provider TEXT NOT NULL DEFAULT '', model TEXT NOT NULL DEFAULT '',
|
|
session_id TEXT NOT NULL DEFAULT '', agent_active INTEGER NOT NULL DEFAULT 0, transcript TEXT NOT NULL DEFAULT '',
|
|
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT, PRIMARY KEY(run_id,sequence));
|
|
CREATE INDEX IF NOT EXISTS run_latest ON pipeline_runs(id DESC);
|
|
CREATE INDEX IF NOT EXISTS run_steps_session ON run_steps(session_id);
|
|
CREATE TABLE IF NOT EXISTS step_notifications (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, receipts_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_file_changes (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, changes_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_control (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, evidence_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_artifacts (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, artifacts_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_verdicts (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, verdict_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_session_exports (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS step_usage (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, usage_json TEXT NOT NULL,
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS run_session_owners (
|
|
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
|
|
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
|
CREATE TABLE IF NOT EXISTS run_retries (
|
|
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
|
|
CREATE TABLE IF NOT EXISTS retry_checkpoints (
|
|
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
|
|
mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL,
|
|
source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := s.migrateRunTitles(); err != nil {
|
|
return err
|
|
}
|
|
for _, column := range []string{"cancel_requested", "cleanup_unconfirmed"} {
|
|
if err := s.addColumnIfMissing("pipeline_runs", column, "INTEGER NOT NULL DEFAULT 0"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_repairs (id INTEGER PRIMARY KEY, run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, export_json TEXT NOT NULL, FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE); CREATE INDEX IF NOT EXISTS session_export_repairs_visit ON session_export_repairs(run_id,sequence,id);`); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("pipeline_runs", "output_format", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("pipeline_runs", "notify_mode", "TEXT NOT NULL DEFAULT 'auto'"); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("pipeline_runs", "environment_json", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
|
return err
|
|
}
|
|
for _, column := range []string{"failure_kind", "failure_reason"} {
|
|
if err := s.addColumnIfMissing("pipeline_runs", column, "TEXT NOT NULL DEFAULT ''"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := s.dropResourceSchema(); err != nil {
|
|
return err
|
|
}
|
|
if err := s.migrateRunModels(); err != nil {
|
|
return err
|
|
}
|
|
return s.initSessionExportJobs()
|
|
}
|
|
|
|
// dropResourceSchema removes the retired resource pool, ownership, wait queue
|
|
// and recovery tables, plus the queue snapshot column. It is idempotent: a
|
|
// database that predates the feature or was already migrated opens cleanly,
|
|
// while an older database loses only the removed feature's state.
|
|
func (s *Store) dropResourceSchema() error {
|
|
for _, table := range []string{"resource_owners", "resource_waiters", "resource_pools", "resource_recovery_audit"} {
|
|
if _, err := s.db.Exec("DROP TABLE IF EXISTS " + table); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return s.dropColumnIfMissing("pipeline_runs", "resource_wait")
|
|
}
|
|
|
|
// dropColumnIfMissing removes a retired column only when it is still present, so
|
|
// repeated Open calls and databases created after the removal stay idempotent.
|
|
func (s *Store) dropColumnIfMissing(table, column string) error {
|
|
rows, err := s.db.Query("PRAGMA table_info(" + table + ")")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
found := false
|
|
for rows.Next() {
|
|
var cid, notnull, pk int
|
|
var name, ctype string
|
|
var dflt sql.NullString
|
|
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
if name == column {
|
|
found = true
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
rows.Close()
|
|
if !found {
|
|
return nil
|
|
}
|
|
_, err = s.db.Exec("ALTER TABLE " + table + " DROP COLUMN " + column)
|
|
return err
|
|
}
|
|
|
|
// migrateRunModels adds the launch-time model-override columns to a database
|
|
// created before per-run overrides existed. Each column is added only when
|
|
// missing, so an upgrade is idempotent across repeated Open calls and existing
|
|
// rows keep their defaults (no model override, no recorded step model).
|
|
func (s *Store) migrateRunModels() error {
|
|
for _, column := range [][3]string{
|
|
{"pipeline_runs", "model", "TEXT NOT NULL DEFAULT ''"},
|
|
{"pipeline_runs", "stage_models", "TEXT NOT NULL DEFAULT '{}'"},
|
|
{"run_steps", "provider", "TEXT NOT NULL DEFAULT ''"},
|
|
{"run_steps", "model", "TEXT NOT NULL DEFAULT ''"},
|
|
} {
|
|
if err := s.addColumnIfMissing(column[0], column[1], column[2]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// migrateRunTitles adds the title columns to a database created before run
|
|
// titles existed. Adding a column with a default leaves every existing row valid
|
|
// and untouched, so old runs still list.
|
|
func (s *Store) migrateRunTitles() error {
|
|
for _, column := range [][2]string{
|
|
{"title", "TEXT NOT NULL DEFAULT ''"},
|
|
{"title_source", "TEXT NOT NULL DEFAULT ''"},
|
|
{"title_model", "TEXT NOT NULL DEFAULT ''"},
|
|
} {
|
|
if err := s.addColumnIfMissing("pipeline_runs", column[0], column[1]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addColumnIfMissing adds a column only when the table predates it, making an
|
|
// upgrade idempotent across repeated Open calls without rewriting history.
|
|
func (s *Store) addColumnIfMissing(table, column, definition string) error {
|
|
rows, err := s.db.Query("PRAGMA table_info(" + table + ")")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var cid, notnull, pk int
|
|
var name, ctype string
|
|
var dflt sql.NullString
|
|
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
|
return err
|
|
}
|
|
if name == column {
|
|
return nil
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
_, err = s.db.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " " + definition)
|
|
return err
|
|
}
|
|
|
|
// RestartStep marks a visit that a stopped server had begun but had not yet
|
|
// handed to any harness session as interrupted, so its run can begin the same
|
|
// stage again under a new sequence. Nothing was admitted, so nothing replays.
|
|
func (s *Store) RestartStep(ctx context.Context, id int64, sequence int, diagnostic string) error {
|
|
return affected(s.db.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE run_id=? AND sequence=? AND status='running' AND session_id=''", diagnostic, id, sequence))
|
|
}
|
|
|
|
func (s *Store) RecoverRuns(ctx context.Context) error {
|
|
return s.RecoverRunsKeeping(ctx, nil, nil)
|
|
}
|
|
|
|
// RecoverRunsKeeping is RecoverRuns for a server that reattaches work across a
|
|
// restart: runs in keep are left exactly as the previous process left them (the
|
|
// runner continues them), and a run in reasons is interrupted with that reason
|
|
// instead of the generic one, so the operator sees why it was not reattached.
|
|
func (s *Store) RecoverRunsKeeping(ctx context.Context, keep map[int64]bool, reasons map[int64]string) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
kept, keptArgs := "", []any{}
|
|
for id := range keep {
|
|
if kept != "" {
|
|
kept += ","
|
|
}
|
|
kept += "?"
|
|
keptArgs = append(keptArgs, id)
|
|
}
|
|
stepFilter, runFilter := "", ""
|
|
if kept != "" {
|
|
stepFilter = " AND run_id NOT IN (" + kept + ")"
|
|
runFilter = " AND id NOT IN (" + kept + ")"
|
|
}
|
|
// No replay: pending visits never began. A cleanup agent that was still
|
|
// active is marked interrupted, and its run is interrupted below.
|
|
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)"+stepFilter, append([]any{protocol.StepStatusWaiting}, keptArgs...)...); err != nil {
|
|
return err
|
|
}
|
|
for id, reason := range reasons {
|
|
if keep[id] {
|
|
continue
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET diagnostic=? WHERE run_id=? AND status='interrupted' AND sequence=(SELECT MAX(sequence) FROM run_steps WHERE run_id=?)", "Server restarted; "+reason, id, id); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,failure_kind='unexpected',failure_reason=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", "Server restarted; "+reason+"; run was not resumed", id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',failure_kind=CASE WHEN cancel_requested=1 THEN '' ELSE 'unexpected' END,failure_reason=CASE WHEN cancel_requested=1 THEN '' ELSE 'Server stopped; owned sessions stopped; run was not resumed' END,finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil {
|
|
return err
|
|
}
|
|
// An export checkpoint is a paused handoff, not an interrupted model call.
|
|
// Also migrate the earlier failed-gate representation before clients attach.
|
|
if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',finished_at=NULL,diagnostic='Session export pending; computed visit retained'
|
|
WHERE status IN ('queued','running','failed','interrupted','waiting') AND cancel_requested=0 AND cleanup_unconfirmed=0
|
|
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
|
|
WHERE g.run_id=pipeline_runs.id AND s.status IN ('completed','failed') AND s.agent_active=0
|
|
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.sequence>g.sequence OR a.agent_active<>0)))`); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|