Files
pipelines/internal/pipeline/resources.go
T

760 lines
28 KiB
Go

package pipeline
import (
"context"
"crypto/rand"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
const maxRecoveryText = 512
func boundedRecoveryText(value string) string {
value = strings.TrimSpace(value)
return value
}
// InspectResource exposes only bounded state; capability tokens never leave this
// package or enter a protocol value.
func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) {
if !ValidResourceKey(resource) {
return protocol.ResourceInspection{}, fmt.Errorf("invalid resource")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}}
var runID int64
var stage string
var seq, cleanup int
err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup)
if err == nil {
var active int
if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
return out, err
}
out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0}
} else if err != sql.ErrNoRows {
return out, err
}
rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource)
if err != nil {
return out, err
}
for rows.Next() {
var w protocol.ResourceWaiter
if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil {
rows.Close()
return out, err
}
out.Waiters = append(out.Waiters, w)
}
if err := rows.Err(); err != nil {
rows.Close()
return out, err
}
rows.Close()
audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource)
if err != nil {
return out, err
}
defer audits.Close()
for audits.Next() {
var a protocol.ResourceRecoveryAudit
a.Resource = resource
if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil {
audits.Close()
return out, err
}
out.Audits = append(out.Audits, a)
}
return out, audits.Err()
}
// ReconcileResource is the sole operator recovery mutation. The persisted
// capability is compared only inside this authoritative store operation.
func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error {
if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) {
return fmt.Errorf("invalid resource recovery identity")
}
if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) {
return fmt.Errorf("positive cleanup/restoration evidence is required")
}
if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText {
return fmt.Errorf("bounded actor is required")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
var token, stage string
var runID int64
var seq, cleanup int
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup)
if err == sql.ErrNoRows {
var count int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil {
return err
}
if count > 0 {
return nil
}
return fmt.Errorf("resource owner not found")
}
if err != nil {
return err
}
if runID != request.RunID || stage != request.Stage || seq != request.Sequence {
return fmt.Errorf("resource owner visit is stale")
}
if cleanup != 0 {
return fmt.Errorf("resource owner is not quarantined")
}
var status string
if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil {
return err
}
if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning {
return fmt.Errorf("owning run is still active")
}
var active int
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
return err
}
if active != 0 {
return fmt.Errorf("owning agent is still active")
}
evidence := boundedRecoveryText(request.Evidence.Summary)
if request.Evidence.NoMutatingAction {
evidence = boundedRecoveryText("no mutating action occurred; " + evidence)
}
evidenceHash := sha256.Sum256([]byte(evidence))
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil {
return err
}
owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token}
if _, err := deleteResourceOwner(ctx, tx, owner); err != nil {
return err
}
next, err := s.handoffResourceTx(ctx, tx, request.Resource)
if err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
if next.token != "" {
s.resourceWaiters[next.ResourceVisit] <- next
}
return nil
}
func validRecoveryEvidence(value string) bool {
value = strings.TrimSpace(value)
if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) {
return false
}
if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") {
return false
}
if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
return false
}
for _, r := range value {
if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') {
return false
}
}
return true
}
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
type ResourceVisit struct {
Resource string
RunID int64
Stage string
Sequence int
}
// ResourceLease is an opaque capability. Its credential is deliberately not exported
// or serializable, and must never be put in model input or public wire metadata.
type ResourceLease struct {
ResourceVisit
Target *protocol.ResourceTarget
pool string
token string
}
const maxPoolMembers = 32
const maxTargetMetadata = 16
const maxTargetMetadataKey = 128
const maxTargetValue = 128
func validateTargetMetadata(metadata map[string]string) error {
if len(metadata) > maxTargetMetadata {
return fmt.Errorf("too many target metadata fields")
}
for k, v := range metadata {
if !identifier.MatchString(k) || len(k) > maxTargetMetadataKey || len(v) > maxTargetValue || !utf8.ValidString(v) {
return fmt.Errorf("invalid target metadata")
}
}
return nil
}
func validateResourcePool(name string, members []protocol.ResourcePoolMember) error {
if !ValidResourceKey(name) || len(members) == 0 || len(members) > maxPoolMembers {
return fmt.Errorf("invalid resource pool")
}
seen := make(map[string]bool, len(members))
for i := range members {
member := &members[i]
if !ValidResourceKey(member.Member) || seen[member.Member] {
return fmt.Errorf("invalid pool member")
}
if err := validateTargetMetadata(member.Metadata); err != nil {
return err
}
seen[member.Member] = true
}
return nil
}
// ConfigureResourcePool replaces a pool declaration atomically. Existing
// owners keep their concrete member; queued requests are evaluated against the
// new declaration on the next release and are never silently transferred.
func (s *Store) ConfigureResourcePool(ctx context.Context, pool protocol.ResourcePool) error {
if err := validateResourcePool(pool.Name, pool.Members); err != nil {
return err
}
encoded, err := json.Marshal(pool.Members)
if err != nil {
return err
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
_, err = s.db.ExecContext(ctx, `INSERT INTO resource_pools(name,members_json) VALUES(?,?) ON CONFLICT(name) DO UPDATE SET members_json=excluded.members_json,updated_at=CURRENT_TIMESTAMP`, pool.Name, string(encoded))
return err
}
// InspectResourcePool returns the declared pool, including only bounded,
// non-secret member metadata and status.
func (s *Store) InspectResourcePool(ctx context.Context, name string) (protocol.ResourcePool, error) {
if !ValidResourceKey(name) {
return protocol.ResourcePool{}, fmt.Errorf("invalid resource pool")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
members, err := s.resourcePool(ctx, s.db, name)
if err != nil {
return protocol.ResourcePool{}, err
}
return protocol.ResourcePool{Name: name, Members: members}, nil
}
func (s *Store) resourcePool(ctx context.Context, tx interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}, name string) ([]protocol.ResourcePoolMember, error) {
var encoded string
if err := tx.QueryRowContext(ctx, "SELECT members_json FROM resource_pools WHERE name=?", name).Scan(&encoded); err != nil {
return nil, err
}
var members []protocol.ResourcePoolMember
if err := json.Unmarshal([]byte(encoded), &members); err != nil {
return nil, fmt.Errorf("invalid persisted pool: %w", err)
}
if err := validateResourcePool(name, members); err != nil {
return nil, fmt.Errorf("invalid persisted pool: %w", err)
}
return members, nil
}
func poolTarget(member protocol.ResourcePoolMember) *protocol.ResourceTarget {
metadata := make(map[string]string, len(member.Metadata))
for k, v := range member.Metadata {
metadata[k] = v
}
return &protocol.ResourceTarget{Member: member.Member, Metadata: metadata}
}
// RequestResourcePool selects one free eligible concrete member while holding
// the same authority as direct resource requests.
func (s *Store) RequestResourcePool(ctx context.Context, pool string, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
if !ValidResourceKey(pool) || visit.Resource != pool || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource pool visit")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
if ch := s.resourceWaiters[visit]; ch != nil {
wait, err := s.resourceWait(ctx, visit.RunID)
return ResourceLease{}, ch, wait, err
}
credential := make([]byte, 32)
if _, err := rand.Read(credential); err != nil {
return ResourceLease{}, nil, nil, err
}
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return ResourceLease{}, nil, nil, err
}
defer tx.Rollback()
var status string
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
return ResourceLease{}, nil, nil, err
}
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
}
var existing ResourceLease
var existingPool string
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool)
if err == nil {
if existing.RunID != visit.RunID || existing.Stage != visit.Stage || existing.Sequence != visit.Sequence {
return ResourceLease{}, nil, nil, fmt.Errorf("resource owner visit is stale")
}
if existingPool != pool {
return ResourceLease{}, nil, nil, fmt.Errorf("resource owner pool visit is stale")
}
members, poolErr := s.resourcePool(ctx, tx, pool)
if poolErr != nil {
return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", poolErr)
}
for _, member := range members {
if member.Member == existing.Resource {
existing.ResourceVisit = visit
existing.pool = pool
existing.Target = poolTarget(member)
return existing, nil, nil, nil
}
}
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
}
if err != sql.ErrNoRows {
return ResourceLease{}, nil, nil, err
}
// A queued visit may be re-entered after the process that owned its live
// channel was restarted. Reattach to the durable waiter instead of trying
// to insert a second reservation.
var persistedPool string
err = tx.QueryRowContext(ctx, "SELECT pool_name FROM resource_waiters WHERE run_id=? AND stage_key=? AND sequence=?", visit.RunID, visit.Stage, visit.Sequence).Scan(&persistedPool)
if err == nil {
if persistedPool != pool {
return ResourceLease{}, nil, nil, fmt.Errorf("resource waiter pool visit is stale")
}
wait, waitErr := s.resourceWait(ctx, visit.RunID)
if waitErr != nil {
return ResourceLease{}, nil, nil, waitErr
}
if s.resourceWaiters == nil {
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
}
ch := make(chan ResourceLease, 1)
s.resourceWaiters[visit] = ch
return ResourceLease{}, ch, wait, nil
}
if err != sql.ErrNoRows {
return ResourceLease{}, nil, nil, err
}
members, err := s.resourcePool(ctx, tx, pool)
if err != nil {
return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", err)
}
for _, member := range members {
if !member.Enabled || member.Quarantined {
continue
}
var count int
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); err != nil {
return ResourceLease{}, nil, nil, err
}
if count != 0 {
continue
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, member.Member, visit.RunID, visit.Stage, visit.Sequence, lease.token, pool); err != nil {
return ResourceLease{}, nil, nil, err
}
lease.Resource = member.Member
lease.pool = pool
lease.Target = poolTarget(member)
if err := tx.Commit(); err != nil {
return ResourceLease{}, nil, nil, err
}
return lease, nil, nil, nil
}
wait := &protocol.ResourceWait{Resource: pool, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
encoded, err := json.Marshal(wait)
if err != nil {
return ResourceLease{}, nil, nil, err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,pool_name,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?,?)`, pool, pool, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
return ResourceLease{}, nil, nil, err
}
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
return ResourceLease{}, nil, nil, err
}
if err := tx.Commit(); err != nil {
return ResourceLease{}, nil, nil, err
}
if s.resourceWaiters == nil {
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
}
ch := make(chan ResourceLease, 1)
s.resourceWaiters[visit] = ch
return ResourceLease{}, ch, wait, nil
}
// RequestResource acquires immediately or registers one FIFO waiter/channel. The
// mutex covers only short DB transactions and channel bookkeeping, never execution.
func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit")
}
if ch := s.resourceWaiters[visit]; ch != nil {
wait, err := s.resourceWait(ctx, visit.RunID)
return ResourceLease{}, ch, wait, err
}
credential := make([]byte, 32)
if _, err := rand.Read(credential); err != nil {
return ResourceLease{}, nil, nil, err
}
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return ResourceLease{}, nil, nil, err
}
defer tx.Rollback()
var status string
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
return ResourceLease{}, nil, nil, err
}
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
}
var existing ResourceLease
var existingPool string
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool)
if err == nil {
if existingPool != "" {
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns a pooled resource visit")
}
if existing.ResourceVisit != visit {
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
}
return existing, nil, nil, nil
}
if err != sql.ErrNoRows {
return ResourceLease{}, nil, nil, err
}
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name)
VALUES(?,?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, "")
if err != nil {
return ResourceLease{}, nil, nil, err
}
n, err := result.RowsAffected()
if err != nil {
return ResourceLease{}, nil, nil, err
}
if n == 1 {
return lease, nil, nil, tx.Commit()
}
wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
encoded, err := json.Marshal(wait)
if err != nil {
return ResourceLease{}, nil, nil, err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
return ResourceLease{}, nil, nil, err
}
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
return ResourceLease{}, nil, nil, err
}
if err := tx.Commit(); err != nil {
return ResourceLease{}, nil, nil, err
}
if s.resourceWaiters == nil {
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
}
ch := make(chan ResourceLease, 1)
s.resourceWaiters[visit] = ch
return ResourceLease{}, ch, wait, nil
}
func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) {
var wait protocol.ResourceWait
err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince)
if err == sql.ErrNoRows {
return nil, nil
}
return &wait, err
}
// CancelResource removes a queued visit, or returns a concurrently reserved grant
// to the queue before any execution is allowed. It cannot cancel another visit.
func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
defer delete(s.resourceWaiters, visit)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence)
if err != nil {
return err
}
n, err := deleted.RowsAffected()
if err != nil {
return err
}
if n > 0 {
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil {
return err
}
}
ch := s.resourceWaiters[visit]
// A grant is buffered while this registry entry exists. Cancellation can
// reclaim only that not-yet-accepted capability, never an executing owner.
var grant ResourceLease
if ch != nil {
select {
case grant = <-ch:
default:
}
}
if err := tx.Commit(); err != nil {
return err
}
if grant.token != "" {
if err := s.SetResourceCleanup(ctx, grant, true); err != nil {
return err
}
return s.releaseResourceLocked(ctx, grant)
}
return nil
}
// AcceptResource removes the channel exactly once. Cancellation is rechecked by
// the runner before BeginStep; a canceled accepted grant is released as unused.
func (s *Store) AcceptResource(lease ResourceLease) {
s.resourceMu.Lock()
delete(s.resourceWaiters, lease.ResourceVisit)
s.resourceMu.Unlock()
}
// HasResourceOwner is used only for orphan/restart reconciliation, never for a
// resource-free stage visit.
func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) {
var exists bool
err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists)
return exists, err
}
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
return err
}
// SetResourceCleanup records positive quiescence durably before release. False
// quarantines the owner; neither age nor another run can remove the lease.
func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error {
if lease.pool != "" && (lease.Target == nil || !ValidResourceKey(lease.Target.Member)) {
return fmt.Errorf("invalid pooled resource lease")
}
diagnostic := ""
if !confirmed {
diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation"
}
resource := lease.Resource
if lease.pool != "" {
resource = lease.Target.Member
}
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
}
// ReleaseResource requires the complete capability AND positive cleanup evidence.
// Mismatches and repeated releases are harmless, including after transfer.
func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
return s.releaseResourceLocked(ctx, lease)
}
func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
ownerResource := lease.Resource
if lease.pool != "" {
if lease.Target == nil || !ValidResourceKey(lease.Target.Member) {
return fmt.Errorf("invalid pooled resource lease")
}
ownerResource = lease.Target.Member
}
lease.Resource = ownerResource
n, err := deleteResourceOwner(ctx, tx, lease)
if err != nil {
return err
}
if n == 0 {
return nil
}
next, err := s.handoffResourceTx(ctx, tx, ownerResource)
if err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
if next.token != "" {
s.resourceWaiters[next.ResourceVisit] <- next
}
return nil
}
// handoffResourceTx is shared by ordinary release and inspected recovery. It
// validates the in-memory waiter before changing durable ownership so a stale
// server cannot grant a capability to a runner that no longer exists.
func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) {
var next ResourceLease
next.Resource = resource
rows, err := tx.QueryContext(ctx, `SELECT queue_order,resource_key,pool_name,run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? OR pool_name<>'' ORDER BY queue_order`, resource)
if err != nil {
return ResourceLease{}, err
}
defer rows.Close()
var selected *protocol.ResourceTarget
var queue int64
var waiterResource string
for rows.Next() {
var candidate ResourceLease
var candidatePool string
if err := rows.Scan(&queue, &waiterResource, &candidatePool, &candidate.RunID, &candidate.Stage, &candidate.Sequence, &candidate.token); err != nil {
return ResourceLease{}, err
}
if candidatePool != "" {
candidate.Resource = candidatePool
candidate.pool = candidatePool
} else {
candidate.Resource = resource
}
if candidatePool == "" {
if waiterResource != resource {
continue
}
selected = &protocol.ResourceTarget{Member: resource}
} else {
members, e := s.resourcePool(ctx, tx, candidatePool)
if e != nil {
continue
}
found := false
for _, member := range members {
if !member.Enabled || member.Quarantined {
continue
}
var count int
if e = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); e != nil {
return ResourceLease{}, e
}
if count == 0 {
selected, found = poolTarget(member), true
break
}
}
if !found {
continue
}
}
if s.resourceWaiters[candidate.ResourceVisit] == nil {
if candidatePool == "" {
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
continue
}
next = candidate
next.Target = selected
break
}
if err := rows.Err(); err != nil {
return ResourceLease{}, err
}
if next.token == "" {
return ResourceLease{}, nil
}
ownerResource := next.Resource
if next.pool != "" {
ownerResource = next.Target.Member
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, ownerResource, next.RunID, next.Stage, next.Sequence, next.token, next.pool); err != nil {
return ResourceLease{}, err
}
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE queue_order=?`, queue); err != nil {
return ResourceLease{}, err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return ResourceLease{}, err
}
if next.pool == "" {
next.Resource = ownerResource
}
return next, nil
}
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
// recoverResourceOwners runs only at startup, after pending waiters have been
// interrupted and removed. Every reclamation uses the persisted capability;
// absent cleanup evidence is never replaced with a timestamp-based expiry.
func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error {
rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1")
if err != nil {
return err
}
var leases []ResourceLease
for rows.Next() {
var lease ResourceLease
if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil {
rows.Close()
return err
}
leases = append(leases, lease)
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
for _, lease := range leases {
if _, err := deleteResourceOwner(ctx, tx, lease); err != nil {
return err
}
}
_, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'")
return err
}