Files
pipelines/internal/pipeline/store.go
T

345 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package pipeline
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"os"
"path/filepath"
"strings"
"sync"
_ "modernc.org/sqlite"
)
type Store struct {
db *sql.DB
dir string
path string
mu sync.Mutex
runnerLock *os.File
runnerClaimed bool
done chan struct{}
closeOnce sync.Once
}
// openDB opens one SQLite connection pool for a DSN. Every open path shares it
// so the connection count and pool settings cannot drift between the writable,
// read-only and maintenance stores.
func openDB(dsn string) (*sql.DB, error) {
db, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, err
}
db.SetMaxOpenConns(1)
return db, nil
}
func Open(path string) (*Store, error) {
db, err := openDB(path)
if err != nil {
return nil, err
}
_, err = db.Exec(`PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;
CREATE TABLE IF NOT EXISTS pipelines (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS pipeline_versions (id INTEGER PRIMARY KEY, pipeline_id INTEGER NOT NULL REFERENCES pipelines(id), version INTEGER NOT NULL, source TEXT NOT NULL, content_hash TEXT NOT NULL, graph_json TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(pipeline_id, version));
CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NULL, source TEXT NOT NULL, diagnostic TEXT NOT NULL, version_id INTEGER REFERENCES pipeline_versions(id), created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);`)
if err != nil {
db.Close()
return nil, err
}
s := &Store{db: db, dir: filepath.Dir(path), path: path, done: make(chan struct{})}
if err := s.initRuns(); err != nil {
db.Close()
return nil, err
}
if err := s.initSearch(); err != nil {
db.Close()
return nil, err
}
if err := s.initRunNotifications(); err != nil {
db.Close()
return nil, err
}
if err := s.initSchedules(); err != nil {
db.Close()
return nil, err
}
if err := s.initOperatorSessions(); err != nil {
db.Close()
return nil, err
}
if err := s.initServerSettings(); err != nil {
db.Close()
return nil, err
}
return s, nil
}
func (s *Store) Close() error {
s.closeOnce.Do(func() {
if s.done != nil {
close(s.done)
}
})
err := s.db.Close()
s.mu.Lock()
defer s.mu.Unlock()
if s.runnerLock != nil {
_ = s.runnerLock.Close()
s.runnerLock = nil
}
return err
}
// Done lets runtime workers stop when an embedded caller closes its store even
// if the caller supplied a background context. It is not a shutdown substitute
// for active execution: callers must still cancel and drain runs before Close.
func (s *Store) Done() <-chan struct{} { return s.done }
// OpenReadOnly opens an existing database without running schema DDL or
// changing journal mode, so a live database can be queried read-only while its
// server keeps writing. It fails when the file does not exist. It is used by
// tests that inspect a real database and is never part of server startup.
func OpenReadOnly(path string) (*Store, error) {
if _, err := os.Stat(path); err != nil {
return nil, err
}
db, err := openDB("file:" + path + "?mode=ro")
if err != nil {
return nil, err
}
if _, err := db.Exec("PRAGMA busy_timeout=5000"); err != nil {
db.Close()
return nil, err
}
return &Store{db: db, dir: filepath.Dir(path), path: path}, nil
}
// OpenMaintenance opens an existing database read-write for an explicit
// operator maintenance action such as retention apply. It never creates a file
// and never runs schema DDL or changes the journal mode, so it cannot silently
// create or migrate a live database. It uses an immediate transaction lock and
// a longer busy timeout, so a concurrent server write produces an honest busy
// failure instead of a hang. The deployed server is never started or changed.
func OpenMaintenance(path string) (*Store, error) {
if _, err := os.Stat(path); err != nil {
return nil, err
}
db, err := openDB("file:" + path + "?_txlock=immediate")
if err != nil {
return nil, err
}
if _, err := db.Exec("PRAGMA busy_timeout=30000; PRAGMA foreign_keys=ON"); err != nil {
db.Close()
return nil, err
}
return &Store{db: db, dir: filepath.Dir(path), path: path}, nil
}
// Directory is the resolved directory holding the database, used to place
// adjacent server state such as stage artifacts.
func (s *Store) Directory() string { return s.dir }
func (s *Store) List(ctx context.Context) ([]protocol.PipelineSummary, error) {
rows, err := s.db.QueryContext(ctx, `SELECT p.id, p.name, v.version,
json_array_length(v.graph_json, '$.stages'),
strftime('%Y-%m-%dT%H:%M:%SZ', p.created_at), strftime('%Y-%m-%dT%H:%M:%SZ', v.created_at),
v.graph_json,
(SELECT COUNT(*) FROM saved_schedules WHERE pipeline_id=p.id AND deleted=0),
(SELECT COUNT(*) FROM saved_schedules WHERE pipeline_id=p.id AND deleted=0 AND enabled=1)
FROM pipelines p JOIN pipeline_versions v ON v.pipeline_id=p.id
WHERE v.version=(SELECT MAX(version) FROM pipeline_versions WHERE pipeline_id=p.id)
ORDER BY p.name, p.id`)
if err != nil {
return nil, err
}
defer rows.Close()
result := []protocol.PipelineSummary{}
for rows.Next() {
var item protocol.PipelineSummary
var graphJSON string
if err := rows.Scan(&item.ID, &item.Name, &item.Version, &item.Stages, &item.CreatedAt, &item.UpdatedAt, &graphJSON, &item.ScheduleCount, &item.EnabledScheduleCount); err != nil {
return nil, err
}
var shape protocol.Graph
item.Scheduled = item.ScheduleCount > 0
if err := json.Unmarshal([]byte(graphJSON), &shape); err == nil && len(shape.Stages) > 0 {
// A list needs the drawing, not the definition: strip prompts,
// configs, tools, limits, notifications and checks so one list reply
// cannot leak a whole pipeline source or grow into a large payload.
shape.Settings = nil
for index := range shape.Stages {
stage := &shape.Stages[index]
stage.Prompt = ""
stage.Output = ""
stage.Configs = nil
stage.Blocks = nil
stage.Memory = nil
stage.Notify = nil
stage.Limits = nil
}
item.Shape = &shape
}
result = append(result, item)
}
return result, rows.Err()
}
// Get returns a consistent immutable version and the available version history.
// Version zero selects the latest accepted version.
func (s *Store) Get(ctx context.Context, id int64, version int) (*protocol.PipelineDetail, error) {
if id <= 0 || version < 0 {
return nil, fmt.Errorf("invalid pipeline ID or version")
}
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
detail := &protocol.PipelineDetail{}
err = tx.QueryRowContext(ctx, `SELECT p.id,p.name,strftime('%Y-%m-%dT%H:%M:%SZ',p.created_at),
v.version,strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at)
FROM pipelines p JOIN pipeline_versions v ON v.pipeline_id=p.id
WHERE p.id=? ORDER BY v.version DESC LIMIT 1`, id).
Scan(&detail.ID, &detail.Name, &detail.CreatedAt, &detail.LatestVersion, &detail.UpdatedAt)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("pipeline not found")
}
if err != nil {
return nil, err
}
if version == 0 {
version = detail.LatestVersion
}
var graph string
err = tx.QueryRowContext(ctx, `SELECT version,source,content_hash,graph_json,strftime('%Y-%m-%dT%H:%M:%SZ',created_at)
FROM pipeline_versions WHERE pipeline_id=? AND version=?`, id, version).
Scan(&detail.Version, &detail.Source, &detail.ContentHash, &graph, &detail.VersionCreatedAt)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("pipeline version not found")
}
if err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(graph), &detail.Graph); err != nil {
return nil, err
}
// Normalize legacy terminal aliases for clients without rewriting history.
for i := range detail.Graph.Stages {
s := &detail.Graph.Stages[i]
s.Type = stage.EffectiveType(s.Type)
for outcome, target := range s.Edges {
s.Edges[outcome] = protocol.ResolveTarget(outcome, target)
}
}
detail.Stages = len(detail.Graph.Stages)
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*),COALESCE(SUM(enabled),0) FROM saved_schedules WHERE pipeline_id=? AND deleted=0`, id).Scan(&detail.ScheduleCount, &detail.EnabledScheduleCount); err != nil {
return nil, err
}
detail.Scheduled = detail.ScheduleCount > 0
rows, err := tx.QueryContext(ctx, `SELECT version,json_array_length(graph_json,'$.stages'),strftime('%Y-%m-%dT%H:%M:%SZ',created_at)
FROM pipeline_versions WHERE pipeline_id=? ORDER BY version DESC`, id)
if err != nil {
return nil, err
}
detail.Versions = []protocol.VersionSummary{}
for rows.Next() {
var item protocol.VersionSummary
if err := rows.Scan(&item.Version, &item.Stages, &item.CreatedAt); err != nil {
rows.Close()
return nil, err
}
detail.Versions = append(detail.Versions, item)
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
if err := tx.Commit(); err != nil {
return nil, err
}
return detail, nil
}
// Submit persists both accepted and rejected submissions. Equality is source-byte equality.
func (s *Store) Submit(ctx context.Context, name, source string, createOnly ...bool) (int, error) {
fresh := len(createOnly) > 0 && createOnly[0]
return s.SubmitChecked(ctx, name, source, fresh, 0)
}
// SubmitChecked guards draft edits against concurrent accepted versions atomically.
func (s *Store) SubmitChecked(ctx context.Context, name, source string, createOnly bool, expectedVersion int) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
name = strings.TrimSpace(name)
g, validation := ParseDefinition(source)
if expectedVersion < 0 {
validation = fmt.Errorf("expected version must be nonnegative")
}
if name == "" || len(name) > 200 || strings.ContainsAny(name, "\r\n\x00") {
validation = fmt.Errorf("pipeline name must be 1–200 bytes without line breaks or NUL")
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return 0, err
}
defer tx.Rollback()
var pid int64
var version int
var versionID any
hash := fmt.Sprintf("%x", sha256.Sum256([]byte(source)))
if validation == nil {
_, err = tx.ExecContext(ctx, "INSERT INTO pipelines(name) VALUES (?) ON CONFLICT(name) DO NOTHING", name)
if err != nil {
return 0, err
}
if err = tx.QueryRowContext(ctx, "SELECT id FROM pipelines WHERE name=?", name).Scan(&pid); err != nil {
return 0, err
}
var latestHash string
err = tx.QueryRowContext(ctx, "SELECT version, content_hash FROM pipeline_versions WHERE pipeline_id=? ORDER BY version DESC LIMIT 1", pid).Scan(&version, &latestHash)
if err != nil && err != sql.ErrNoRows {
return 0, err
}
if createOnly && version > 0 {
validation = fmt.Errorf("pipeline name already exists; choose a new name")
} else if expectedVersion > 0 && version != expectedVersion {
validation = fmt.Errorf("draft conflict: expected latest version %d, found %d; review changes before submitting", expectedVersion, version)
} else if hash == latestHash {
validation = fmt.Errorf("no changes")
} else {
version++
encoded, err := json.Marshal(g)
if err != nil {
return 0, err
}
result, err := tx.ExecContext(ctx, "INSERT INTO pipeline_versions(pipeline_id,version,source,content_hash,graph_json) VALUES (?,?,?,?,?)", pid, version, source, hash, string(encoded))
if err != nil {
return 0, err
}
id, err := result.LastInsertId()
if err != nil {
return 0, err
}
versionID = id
}
}
diagnostic := ""
if validation != nil {
diagnostic = validation.Error()
}
if _, err := tx.ExecContext(ctx, "INSERT INTO submissions(name,source,diagnostic,version_id) VALUES (?,?,?,?)", name, source, diagnostic, versionID); err != nil {
return 0, err
}
if err := tx.Commit(); err != nil {
return 0, err
}
return version, validation
}