760 lines
28 KiB
Go
760 lines
28 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
const maxRecoveryText = 512
|
|
|
|
func boundedRecoveryText(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
return value
|
|
}
|
|
|
|
// InspectResource exposes only bounded state; capability tokens never leave this
|
|
// package or enter a protocol value.
|
|
func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) {
|
|
if !ValidResourceKey(resource) {
|
|
return protocol.ResourceInspection{}, fmt.Errorf("invalid resource")
|
|
}
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}}
|
|
var runID int64
|
|
var stage string
|
|
var seq, cleanup int
|
|
err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup)
|
|
if err == nil {
|
|
var active int
|
|
if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
|
|
return out, err
|
|
}
|
|
out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0}
|
|
} else if err != sql.ErrNoRows {
|
|
return out, err
|
|
}
|
|
rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
for rows.Next() {
|
|
var w protocol.ResourceWaiter
|
|
if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil {
|
|
rows.Close()
|
|
return out, err
|
|
}
|
|
out.Waiters = append(out.Waiters, w)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return out, err
|
|
}
|
|
rows.Close()
|
|
audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
defer audits.Close()
|
|
for audits.Next() {
|
|
var a protocol.ResourceRecoveryAudit
|
|
a.Resource = resource
|
|
if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil {
|
|
audits.Close()
|
|
return out, err
|
|
}
|
|
out.Audits = append(out.Audits, a)
|
|
}
|
|
return out, audits.Err()
|
|
}
|
|
|
|
// ReconcileResource is the sole operator recovery mutation. The persisted
|
|
// capability is compared only inside this authoritative store operation.
|
|
func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error {
|
|
if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) {
|
|
return fmt.Errorf("invalid resource recovery identity")
|
|
}
|
|
if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) {
|
|
return fmt.Errorf("positive cleanup/restoration evidence is required")
|
|
}
|
|
if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText {
|
|
return fmt.Errorf("bounded actor is required")
|
|
}
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var token, stage string
|
|
var runID int64
|
|
var seq, cleanup int
|
|
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup)
|
|
if err == sql.ErrNoRows {
|
|
var count int
|
|
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil {
|
|
return err
|
|
}
|
|
if count > 0 {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("resource owner not found")
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if runID != request.RunID || stage != request.Stage || seq != request.Sequence {
|
|
return fmt.Errorf("resource owner visit is stale")
|
|
}
|
|
if cleanup != 0 {
|
|
return fmt.Errorf("resource owner is not quarantined")
|
|
}
|
|
var status string
|
|
if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil {
|
|
return err
|
|
}
|
|
if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning {
|
|
return fmt.Errorf("owning run is still active")
|
|
}
|
|
var active int
|
|
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
|
|
return err
|
|
}
|
|
if active != 0 {
|
|
return fmt.Errorf("owning agent is still active")
|
|
}
|
|
evidence := boundedRecoveryText(request.Evidence.Summary)
|
|
if request.Evidence.NoMutatingAction {
|
|
evidence = boundedRecoveryText("no mutating action occurred; " + evidence)
|
|
}
|
|
evidenceHash := sha256.Sum256([]byte(evidence))
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil {
|
|
return err
|
|
}
|
|
owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token}
|
|
if _, err := deleteResourceOwner(ctx, tx, owner); err != nil {
|
|
return err
|
|
}
|
|
next, err := s.handoffResourceTx(ctx, tx, request.Resource)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if next.token != "" {
|
|
s.resourceWaiters[next.ResourceVisit] <- next
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validRecoveryEvidence(value string) bool {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) {
|
|
return false
|
|
}
|
|
if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") {
|
|
return false
|
|
}
|
|
if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
|
|
return false
|
|
}
|
|
for _, r := range value {
|
|
if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
|
|
type ResourceVisit struct {
|
|
Resource string
|
|
RunID int64
|
|
Stage string
|
|
Sequence int
|
|
}
|
|
|
|
// ResourceLease is an opaque capability. Its credential is deliberately not exported
|
|
// or serializable, and must never be put in model input or public wire metadata.
|
|
type ResourceLease struct {
|
|
ResourceVisit
|
|
Target *protocol.ResourceTarget
|
|
pool string
|
|
token string
|
|
}
|
|
|
|
const maxPoolMembers = 32
|
|
const maxTargetMetadata = 16
|
|
const maxTargetMetadataKey = 128
|
|
const maxTargetValue = 128
|
|
|
|
func validateTargetMetadata(metadata map[string]string) error {
|
|
if len(metadata) > maxTargetMetadata {
|
|
return fmt.Errorf("too many target metadata fields")
|
|
}
|
|
for k, v := range metadata {
|
|
if !identifier.MatchString(k) || len(k) > maxTargetMetadataKey || len(v) > maxTargetValue || !utf8.ValidString(v) {
|
|
return fmt.Errorf("invalid target metadata")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateResourcePool(name string, members []protocol.ResourcePoolMember) error {
|
|
if !ValidResourceKey(name) || len(members) == 0 || len(members) > maxPoolMembers {
|
|
return fmt.Errorf("invalid resource pool")
|
|
}
|
|
seen := make(map[string]bool, len(members))
|
|
for i := range members {
|
|
member := &members[i]
|
|
if !ValidResourceKey(member.Member) || seen[member.Member] {
|
|
return fmt.Errorf("invalid pool member")
|
|
}
|
|
if err := validateTargetMetadata(member.Metadata); err != nil {
|
|
return err
|
|
}
|
|
seen[member.Member] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ConfigureResourcePool replaces a pool declaration atomically. Existing
|
|
// owners keep their concrete member; queued requests are evaluated against the
|
|
// new declaration on the next release and are never silently transferred.
|
|
func (s *Store) ConfigureResourcePool(ctx context.Context, pool protocol.ResourcePool) error {
|
|
if err := validateResourcePool(pool.Name, pool.Members); err != nil {
|
|
return err
|
|
}
|
|
encoded, err := json.Marshal(pool.Members)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
_, err = s.db.ExecContext(ctx, `INSERT INTO resource_pools(name,members_json) VALUES(?,?) ON CONFLICT(name) DO UPDATE SET members_json=excluded.members_json,updated_at=CURRENT_TIMESTAMP`, pool.Name, string(encoded))
|
|
return err
|
|
}
|
|
|
|
// InspectResourcePool returns the declared pool, including only bounded,
|
|
// non-secret member metadata and status.
|
|
func (s *Store) InspectResourcePool(ctx context.Context, name string) (protocol.ResourcePool, error) {
|
|
if !ValidResourceKey(name) {
|
|
return protocol.ResourcePool{}, fmt.Errorf("invalid resource pool")
|
|
}
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
members, err := s.resourcePool(ctx, s.db, name)
|
|
if err != nil {
|
|
return protocol.ResourcePool{}, err
|
|
}
|
|
return protocol.ResourcePool{Name: name, Members: members}, nil
|
|
}
|
|
|
|
func (s *Store) resourcePool(ctx context.Context, tx interface {
|
|
QueryRowContext(context.Context, string, ...any) *sql.Row
|
|
}, name string) ([]protocol.ResourcePoolMember, error) {
|
|
var encoded string
|
|
if err := tx.QueryRowContext(ctx, "SELECT members_json FROM resource_pools WHERE name=?", name).Scan(&encoded); err != nil {
|
|
return nil, err
|
|
}
|
|
var members []protocol.ResourcePoolMember
|
|
if err := json.Unmarshal([]byte(encoded), &members); err != nil {
|
|
return nil, fmt.Errorf("invalid persisted pool: %w", err)
|
|
}
|
|
if err := validateResourcePool(name, members); err != nil {
|
|
return nil, fmt.Errorf("invalid persisted pool: %w", err)
|
|
}
|
|
return members, nil
|
|
}
|
|
|
|
func poolTarget(member protocol.ResourcePoolMember) *protocol.ResourceTarget {
|
|
metadata := make(map[string]string, len(member.Metadata))
|
|
for k, v := range member.Metadata {
|
|
metadata[k] = v
|
|
}
|
|
return &protocol.ResourceTarget{Member: member.Member, Metadata: metadata}
|
|
}
|
|
|
|
// RequestResourcePool selects one free eligible concrete member while holding
|
|
// the same authority as direct resource requests.
|
|
func (s *Store) RequestResourcePool(ctx context.Context, pool string, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
|
|
if !ValidResourceKey(pool) || visit.Resource != pool || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource pool visit")
|
|
}
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
if ch := s.resourceWaiters[visit]; ch != nil {
|
|
wait, err := s.resourceWait(ctx, visit.RunID)
|
|
return ResourceLease{}, ch, wait, err
|
|
}
|
|
credential := make([]byte, 32)
|
|
if _, err := rand.Read(credential); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
var status string
|
|
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
|
|
}
|
|
var existing ResourceLease
|
|
var existingPool string
|
|
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool)
|
|
if err == nil {
|
|
if existing.RunID != visit.RunID || existing.Stage != visit.Stage || existing.Sequence != visit.Sequence {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("resource owner visit is stale")
|
|
}
|
|
if existingPool != pool {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("resource owner pool visit is stale")
|
|
}
|
|
members, poolErr := s.resourcePool(ctx, tx, pool)
|
|
if poolErr != nil {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", poolErr)
|
|
}
|
|
for _, member := range members {
|
|
if member.Member == existing.Resource {
|
|
existing.ResourceVisit = visit
|
|
existing.pool = pool
|
|
existing.Target = poolTarget(member)
|
|
return existing, nil, nil, nil
|
|
}
|
|
}
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
|
|
}
|
|
if err != sql.ErrNoRows {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
// A queued visit may be re-entered after the process that owned its live
|
|
// channel was restarted. Reattach to the durable waiter instead of trying
|
|
// to insert a second reservation.
|
|
var persistedPool string
|
|
err = tx.QueryRowContext(ctx, "SELECT pool_name FROM resource_waiters WHERE run_id=? AND stage_key=? AND sequence=?", visit.RunID, visit.Stage, visit.Sequence).Scan(&persistedPool)
|
|
if err == nil {
|
|
if persistedPool != pool {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("resource waiter pool visit is stale")
|
|
}
|
|
wait, waitErr := s.resourceWait(ctx, visit.RunID)
|
|
if waitErr != nil {
|
|
return ResourceLease{}, nil, nil, waitErr
|
|
}
|
|
if s.resourceWaiters == nil {
|
|
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
|
|
}
|
|
ch := make(chan ResourceLease, 1)
|
|
s.resourceWaiters[visit] = ch
|
|
return ResourceLease{}, ch, wait, nil
|
|
}
|
|
if err != sql.ErrNoRows {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
members, err := s.resourcePool(ctx, tx, pool)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", err)
|
|
}
|
|
for _, member := range members {
|
|
if !member.Enabled || member.Quarantined {
|
|
continue
|
|
}
|
|
var count int
|
|
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if count != 0 {
|
|
continue
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, member.Member, visit.RunID, visit.Stage, visit.Sequence, lease.token, pool); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
lease.Resource = member.Member
|
|
lease.pool = pool
|
|
lease.Target = poolTarget(member)
|
|
if err := tx.Commit(); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
return lease, nil, nil, nil
|
|
}
|
|
wait := &protocol.ResourceWait{Resource: pool, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
|
|
encoded, err := json.Marshal(wait)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,pool_name,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?,?)`, pool, pool, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if s.resourceWaiters == nil {
|
|
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
|
|
}
|
|
ch := make(chan ResourceLease, 1)
|
|
s.resourceWaiters[visit] = ch
|
|
return ResourceLease{}, ch, wait, nil
|
|
}
|
|
|
|
// RequestResource acquires immediately or registers one FIFO waiter/channel. The
|
|
// mutex covers only short DB transactions and channel bookkeeping, never execution.
|
|
func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit")
|
|
}
|
|
if ch := s.resourceWaiters[visit]; ch != nil {
|
|
wait, err := s.resourceWait(ctx, visit.RunID)
|
|
return ResourceLease{}, ch, wait, err
|
|
}
|
|
credential := make([]byte, 32)
|
|
if _, err := rand.Read(credential); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
var status string
|
|
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
|
|
}
|
|
var existing ResourceLease
|
|
var existingPool string
|
|
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool)
|
|
if err == nil {
|
|
if existingPool != "" {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns a pooled resource visit")
|
|
}
|
|
if existing.ResourceVisit != visit {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
|
|
}
|
|
return existing, nil, nil, nil
|
|
}
|
|
if err != sql.ErrNoRows {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name)
|
|
VALUES(?,?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, "")
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if n == 1 {
|
|
return lease, nil, nil, tx.Commit()
|
|
}
|
|
wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
|
|
encoded, err := json.Marshal(wait)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if s.resourceWaiters == nil {
|
|
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
|
|
}
|
|
ch := make(chan ResourceLease, 1)
|
|
s.resourceWaiters[visit] = ch
|
|
return ResourceLease{}, ch, wait, nil
|
|
}
|
|
|
|
func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) {
|
|
var wait protocol.ResourceWait
|
|
err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince)
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
return &wait, err
|
|
}
|
|
|
|
// CancelResource removes a queued visit, or returns a concurrently reserved grant
|
|
// to the queue before any execution is allowed. It cannot cancel another visit.
|
|
func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
defer delete(s.resourceWaiters, visit)
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := deleted.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n > 0 {
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
ch := s.resourceWaiters[visit]
|
|
// A grant is buffered while this registry entry exists. Cancellation can
|
|
// reclaim only that not-yet-accepted capability, never an executing owner.
|
|
var grant ResourceLease
|
|
if ch != nil {
|
|
select {
|
|
case grant = <-ch:
|
|
default:
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if grant.token != "" {
|
|
if err := s.SetResourceCleanup(ctx, grant, true); err != nil {
|
|
return err
|
|
}
|
|
return s.releaseResourceLocked(ctx, grant)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AcceptResource removes the channel exactly once. Cancellation is rechecked by
|
|
// the runner before BeginStep; a canceled accepted grant is released as unused.
|
|
func (s *Store) AcceptResource(lease ResourceLease) {
|
|
s.resourceMu.Lock()
|
|
delete(s.resourceWaiters, lease.ResourceVisit)
|
|
s.resourceMu.Unlock()
|
|
}
|
|
|
|
// HasResourceOwner is used only for orphan/restart reconciliation, never for a
|
|
// resource-free stage visit.
|
|
func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) {
|
|
var exists bool
|
|
err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists)
|
|
return exists, err
|
|
}
|
|
|
|
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
|
|
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
|
|
return err
|
|
}
|
|
|
|
// SetResourceCleanup records positive quiescence durably before release. False
|
|
// quarantines the owner; neither age nor another run can remove the lease.
|
|
func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error {
|
|
if lease.pool != "" && (lease.Target == nil || !ValidResourceKey(lease.Target.Member)) {
|
|
return fmt.Errorf("invalid pooled resource lease")
|
|
}
|
|
diagnostic := ""
|
|
if !confirmed {
|
|
diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation"
|
|
}
|
|
resource := lease.Resource
|
|
if lease.pool != "" {
|
|
resource = lease.Target.Member
|
|
}
|
|
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
|
|
}
|
|
|
|
// ReleaseResource requires the complete capability AND positive cleanup evidence.
|
|
// Mismatches and repeated releases are harmless, including after transfer.
|
|
func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
return s.releaseResourceLocked(ctx, lease)
|
|
}
|
|
|
|
func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
ownerResource := lease.Resource
|
|
if lease.pool != "" {
|
|
if lease.Target == nil || !ValidResourceKey(lease.Target.Member) {
|
|
return fmt.Errorf("invalid pooled resource lease")
|
|
}
|
|
ownerResource = lease.Target.Member
|
|
}
|
|
lease.Resource = ownerResource
|
|
n, err := deleteResourceOwner(ctx, tx, lease)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n == 0 {
|
|
return nil
|
|
}
|
|
next, err := s.handoffResourceTx(ctx, tx, ownerResource)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if next.token != "" {
|
|
s.resourceWaiters[next.ResourceVisit] <- next
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// handoffResourceTx is shared by ordinary release and inspected recovery. It
|
|
// validates the in-memory waiter before changing durable ownership so a stale
|
|
// server cannot grant a capability to a runner that no longer exists.
|
|
func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) {
|
|
var next ResourceLease
|
|
next.Resource = resource
|
|
rows, err := tx.QueryContext(ctx, `SELECT queue_order,resource_key,pool_name,run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? OR pool_name<>'' ORDER BY queue_order`, resource)
|
|
if err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
defer rows.Close()
|
|
var selected *protocol.ResourceTarget
|
|
var queue int64
|
|
var waiterResource string
|
|
for rows.Next() {
|
|
var candidate ResourceLease
|
|
var candidatePool string
|
|
if err := rows.Scan(&queue, &waiterResource, &candidatePool, &candidate.RunID, &candidate.Stage, &candidate.Sequence, &candidate.token); err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
if candidatePool != "" {
|
|
candidate.Resource = candidatePool
|
|
candidate.pool = candidatePool
|
|
} else {
|
|
candidate.Resource = resource
|
|
}
|
|
if candidatePool == "" {
|
|
if waiterResource != resource {
|
|
continue
|
|
}
|
|
selected = &protocol.ResourceTarget{Member: resource}
|
|
} else {
|
|
members, e := s.resourcePool(ctx, tx, candidatePool)
|
|
if e != nil {
|
|
continue
|
|
}
|
|
found := false
|
|
for _, member := range members {
|
|
if !member.Enabled || member.Quarantined {
|
|
continue
|
|
}
|
|
var count int
|
|
if e = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); e != nil {
|
|
return ResourceLease{}, e
|
|
}
|
|
if count == 0 {
|
|
selected, found = poolTarget(member), true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
continue
|
|
}
|
|
}
|
|
if s.resourceWaiters[candidate.ResourceVisit] == nil {
|
|
if candidatePool == "" {
|
|
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
|
|
}
|
|
continue
|
|
}
|
|
next = candidate
|
|
next.Target = selected
|
|
break
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
if next.token == "" {
|
|
return ResourceLease{}, nil
|
|
}
|
|
ownerResource := next.Resource
|
|
if next.pool != "" {
|
|
ownerResource = next.Target.Member
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, ownerResource, next.RunID, next.Stage, next.Sequence, next.token, next.pool); err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE queue_order=?`, queue); err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
if next.pool == "" {
|
|
next.Resource = ownerResource
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
|
|
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return result.RowsAffected()
|
|
}
|
|
|
|
// recoverResourceOwners runs only at startup, after pending waiters have been
|
|
// interrupted and removed. Every reclamation uses the persisted capability;
|
|
// absent cleanup evidence is never replaced with a timestamp-based expiry.
|
|
func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error {
|
|
rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var leases []ResourceLease
|
|
for rows.Next() {
|
|
var lease ResourceLease
|
|
if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
leases = append(leases, lease)
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, lease := range leases {
|
|
if _, err := deleteResourceOwner(ctx, tx, lease); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'")
|
|
return err
|
|
}
|