package pipeline import ( "context" "crypto/rand" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "fmt" "strings" "time" "unicode/utf8" "git.a-wan.net/alan/pipelines/internal/protocol" ) const maxRecoveryText = 512 func boundedRecoveryText(value string) string { value = strings.TrimSpace(value) return value } // InspectResource exposes only bounded state; capability tokens never leave this // package or enter a protocol value. func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) { if !ValidResourceKey(resource) { return protocol.ResourceInspection{}, fmt.Errorf("invalid resource") } s.resourceMu.Lock() defer s.resourceMu.Unlock() out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}} var runID int64 var stage string var seq, cleanup int err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup) if err == nil { var active int if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil { return out, err } out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0} } else if err != sql.ErrNoRows { return out, err } rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource) if err != nil { return out, err } for rows.Next() { var w protocol.ResourceWaiter if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil { rows.Close() return out, err } out.Waiters = append(out.Waiters, w) } if err := rows.Err(); err != nil { rows.Close() return out, err } rows.Close() audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource) if err != nil { return out, err } defer audits.Close() for audits.Next() { var a protocol.ResourceRecoveryAudit a.Resource = resource if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil { audits.Close() return out, err } out.Audits = append(out.Audits, a) } return out, audits.Err() } // ReconcileResource is the sole operator recovery mutation. The persisted // capability is compared only inside this authoritative store operation. func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error { if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) { return fmt.Errorf("invalid resource recovery identity") } if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) { return fmt.Errorf("positive cleanup/restoration evidence is required") } if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText { return fmt.Errorf("bounded actor is required") } s.resourceMu.Lock() defer s.resourceMu.Unlock() tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() var token, stage string var runID int64 var seq, cleanup int err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup) if err == sql.ErrNoRows { var count int if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil { return err } if count > 0 { return nil } return fmt.Errorf("resource owner not found") } if err != nil { return err } if runID != request.RunID || stage != request.Stage || seq != request.Sequence { return fmt.Errorf("resource owner visit is stale") } if cleanup != 0 { return fmt.Errorf("resource owner is not quarantined") } var status string if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil { return err } if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning { return fmt.Errorf("owning run is still active") } var active int if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil { return err } if active != 0 { return fmt.Errorf("owning agent is still active") } evidence := boundedRecoveryText(request.Evidence.Summary) if request.Evidence.NoMutatingAction { evidence = boundedRecoveryText("no mutating action occurred; " + evidence) } evidenceHash := sha256.Sum256([]byte(evidence)) if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil { return err } if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil { return err } owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token} if _, err := deleteResourceOwner(ctx, tx, owner); err != nil { return err } next, err := s.handoffResourceTx(ctx, tx, request.Resource) if err != nil { return err } if err := tx.Commit(); err != nil { return err } if next.token != "" { s.resourceWaiters[next.ResourceVisit] <- next } return nil } func validRecoveryEvidence(value string) bool { value = strings.TrimSpace(value) if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) { return false } if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") { return false } if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") { return false } for _, r := range value { if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') { return false } } return true } // ResourceVisit identifies a pending visit, before it consumes execution accounting. type ResourceVisit struct { Resource string RunID int64 Stage string Sequence int } // ResourceLease is an opaque capability. Its credential is deliberately not exported // or serializable, and must never be put in model input or public wire metadata. type ResourceLease struct { ResourceVisit Target *protocol.ResourceTarget pool string token string } const maxPoolMembers = 32 const maxTargetMetadata = 16 const maxTargetMetadataKey = 128 const maxTargetValue = 128 func validateTargetMetadata(metadata map[string]string) error { if len(metadata) > maxTargetMetadata { return fmt.Errorf("too many target metadata fields") } for k, v := range metadata { if !identifier.MatchString(k) || len(k) > maxTargetMetadataKey || len(v) > maxTargetValue || !utf8.ValidString(v) { return fmt.Errorf("invalid target metadata") } } return nil } func validateResourcePool(name string, members []protocol.ResourcePoolMember) error { if !ValidResourceKey(name) || len(members) == 0 || len(members) > maxPoolMembers { return fmt.Errorf("invalid resource pool") } seen := make(map[string]bool, len(members)) for i := range members { member := &members[i] if !ValidResourceKey(member.Member) || seen[member.Member] { return fmt.Errorf("invalid pool member") } if err := validateTargetMetadata(member.Metadata); err != nil { return err } seen[member.Member] = true } return nil } // ConfigureResourcePool replaces a pool declaration atomically. Existing // owners keep their concrete member; queued requests are evaluated against the // new declaration on the next release and are never silently transferred. func (s *Store) ConfigureResourcePool(ctx context.Context, pool protocol.ResourcePool) error { if err := validateResourcePool(pool.Name, pool.Members); err != nil { return err } encoded, err := json.Marshal(pool.Members) if err != nil { return err } s.resourceMu.Lock() defer s.resourceMu.Unlock() _, err = s.db.ExecContext(ctx, `INSERT INTO resource_pools(name,members_json) VALUES(?,?) ON CONFLICT(name) DO UPDATE SET members_json=excluded.members_json,updated_at=CURRENT_TIMESTAMP`, pool.Name, string(encoded)) return err } // InspectResourcePool returns the declared pool, including only bounded, // non-secret member metadata and status. func (s *Store) InspectResourcePool(ctx context.Context, name string) (protocol.ResourcePool, error) { if !ValidResourceKey(name) { return protocol.ResourcePool{}, fmt.Errorf("invalid resource pool") } s.resourceMu.Lock() defer s.resourceMu.Unlock() members, err := s.resourcePool(ctx, s.db, name) if err != nil { return protocol.ResourcePool{}, err } return protocol.ResourcePool{Name: name, Members: members}, nil } func (s *Store) resourcePool(ctx context.Context, tx interface { QueryRowContext(context.Context, string, ...any) *sql.Row }, name string) ([]protocol.ResourcePoolMember, error) { var encoded string if err := tx.QueryRowContext(ctx, "SELECT members_json FROM resource_pools WHERE name=?", name).Scan(&encoded); err != nil { return nil, err } var members []protocol.ResourcePoolMember if err := json.Unmarshal([]byte(encoded), &members); err != nil { return nil, fmt.Errorf("invalid persisted pool: %w", err) } if err := validateResourcePool(name, members); err != nil { return nil, fmt.Errorf("invalid persisted pool: %w", err) } return members, nil } func poolTarget(member protocol.ResourcePoolMember) *protocol.ResourceTarget { metadata := make(map[string]string, len(member.Metadata)) for k, v := range member.Metadata { metadata[k] = v } return &protocol.ResourceTarget{Member: member.Member, Metadata: metadata} } // RequestResourcePool selects one free eligible concrete member while holding // the same authority as direct resource requests. func (s *Store) RequestResourcePool(ctx context.Context, pool string, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) { if !ValidResourceKey(pool) || visit.Resource != pool || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) { return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource pool visit") } s.resourceMu.Lock() defer s.resourceMu.Unlock() if ch := s.resourceWaiters[visit]; ch != nil { wait, err := s.resourceWait(ctx, visit.RunID) return ResourceLease{}, ch, wait, err } credential := make([]byte, 32) if _, err := rand.Read(credential); err != nil { return ResourceLease{}, nil, nil, err } lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)} tx, err := s.db.BeginTx(ctx, nil) if err != nil { return ResourceLease{}, nil, nil, err } defer tx.Rollback() var status string if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil { return ResourceLease{}, nil, nil, err } if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning { return ResourceLease{}, nil, nil, fmt.Errorf("run is not active") } var existing ResourceLease var existingPool string err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool) if err == nil { if existing.RunID != visit.RunID || existing.Stage != visit.Stage || existing.Sequence != visit.Sequence { return ResourceLease{}, nil, nil, fmt.Errorf("resource owner visit is stale") } if existingPool != pool { return ResourceLease{}, nil, nil, fmt.Errorf("resource owner pool visit is stale") } members, poolErr := s.resourcePool(ctx, tx, pool) if poolErr != nil { return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", poolErr) } for _, member := range members { if member.Member == existing.Resource { existing.ResourceVisit = visit existing.pool = pool existing.Target = poolTarget(member) return existing, nil, nil, nil } } return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit") } if err != sql.ErrNoRows { return ResourceLease{}, nil, nil, err } // A queued visit may be re-entered after the process that owned its live // channel was restarted. Reattach to the durable waiter instead of trying // to insert a second reservation. var persistedPool string err = tx.QueryRowContext(ctx, "SELECT pool_name FROM resource_waiters WHERE run_id=? AND stage_key=? AND sequence=?", visit.RunID, visit.Stage, visit.Sequence).Scan(&persistedPool) if err == nil { if persistedPool != pool { return ResourceLease{}, nil, nil, fmt.Errorf("resource waiter pool visit is stale") } wait, waitErr := s.resourceWait(ctx, visit.RunID) if waitErr != nil { return ResourceLease{}, nil, nil, waitErr } if s.resourceWaiters == nil { s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease) } ch := make(chan ResourceLease, 1) s.resourceWaiters[visit] = ch return ResourceLease{}, ch, wait, nil } if err != sql.ErrNoRows { return ResourceLease{}, nil, nil, err } members, err := s.resourcePool(ctx, tx, pool) if err != nil { return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", err) } for _, member := range members { if !member.Enabled || member.Quarantined { continue } var count int if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); err != nil { return ResourceLease{}, nil, nil, err } if count != 0 { continue } if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, member.Member, visit.RunID, visit.Stage, visit.Sequence, lease.token, pool); err != nil { return ResourceLease{}, nil, nil, err } lease.Resource = member.Member lease.pool = pool lease.Target = poolTarget(member) if err := tx.Commit(); err != nil { return ResourceLease{}, nil, nil, err } return lease, nil, nil, nil } wait := &protocol.ResourceWait{Resource: pool, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)} encoded, err := json.Marshal(wait) if err != nil { return ResourceLease{}, nil, nil, err } if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,pool_name,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?,?)`, pool, pool, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil { return ResourceLease{}, nil, nil, err } if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil { return ResourceLease{}, nil, nil, err } if err := tx.Commit(); err != nil { return ResourceLease{}, nil, nil, err } if s.resourceWaiters == nil { s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease) } ch := make(chan ResourceLease, 1) s.resourceWaiters[visit] = ch return ResourceLease{}, ch, wait, nil } // RequestResource acquires immediately or registers one FIFO waiter/channel. The // mutex covers only short DB transactions and channel bookkeeping, never execution. func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) { s.resourceMu.Lock() defer s.resourceMu.Unlock() if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) { return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit") } if ch := s.resourceWaiters[visit]; ch != nil { wait, err := s.resourceWait(ctx, visit.RunID) return ResourceLease{}, ch, wait, err } credential := make([]byte, 32) if _, err := rand.Read(credential); err != nil { return ResourceLease{}, nil, nil, err } lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)} tx, err := s.db.BeginTx(ctx, nil) if err != nil { return ResourceLease{}, nil, nil, err } defer tx.Rollback() var status string if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil { return ResourceLease{}, nil, nil, err } if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning { return ResourceLease{}, nil, nil, fmt.Errorf("run is not active") } var existing ResourceLease var existingPool string err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool) if err == nil { if existingPool != "" { return ResourceLease{}, nil, nil, fmt.Errorf("run already owns a pooled resource visit") } if existing.ResourceVisit != visit { return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit") } return existing, nil, nil, nil } if err != sql.ErrNoRows { return ResourceLease{}, nil, nil, err } result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, "") if err != nil { return ResourceLease{}, nil, nil, err } n, err := result.RowsAffected() if err != nil { return ResourceLease{}, nil, nil, err } if n == 1 { return lease, nil, nil, tx.Commit() } wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)} encoded, err := json.Marshal(wait) if err != nil { return ResourceLease{}, nil, nil, err } if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil { return ResourceLease{}, nil, nil, err } if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil { return ResourceLease{}, nil, nil, err } if err := tx.Commit(); err != nil { return ResourceLease{}, nil, nil, err } if s.resourceWaiters == nil { s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease) } ch := make(chan ResourceLease, 1) s.resourceWaiters[visit] = ch return ResourceLease{}, ch, wait, nil } func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) { var wait protocol.ResourceWait err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince) if err == sql.ErrNoRows { return nil, nil } return &wait, err } // CancelResource removes a queued visit, or returns a concurrently reserved grant // to the queue before any execution is allowed. It cannot cancel another visit. func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error { s.resourceMu.Lock() defer s.resourceMu.Unlock() defer delete(s.resourceWaiters, visit) tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence) if err != nil { return err } n, err := deleted.RowsAffected() if err != nil { return err } if n > 0 { if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil { return err } } ch := s.resourceWaiters[visit] // A grant is buffered while this registry entry exists. Cancellation can // reclaim only that not-yet-accepted capability, never an executing owner. var grant ResourceLease if ch != nil { select { case grant = <-ch: default: } } if err := tx.Commit(); err != nil { return err } if grant.token != "" { if err := s.SetResourceCleanup(ctx, grant, true); err != nil { return err } return s.releaseResourceLocked(ctx, grant) } return nil } // AcceptResource removes the channel exactly once. Cancellation is rechecked by // the runner before BeginStep; a canceled accepted grant is released as unused. func (s *Store) AcceptResource(lease ResourceLease) { s.resourceMu.Lock() delete(s.resourceWaiters, lease.ResourceVisit) s.resourceMu.Unlock() } // HasResourceOwner is used only for orphan/restart reconciliation, never for a // resource-free stage visit. func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) { var exists bool err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists) return exists, err } func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error { _, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID) return err } // SetResourceCleanup records positive quiescence durably before release. False // quarantines the owner; neither age nor another run can remove the lease. func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error { if lease.pool != "" && (lease.Target == nil || !ValidResourceKey(lease.Target.Member)) { return fmt.Errorf("invalid pooled resource lease") } diagnostic := "" if !confirmed { diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation" } resource := lease.Resource if lease.pool != "" { resource = lease.Target.Member } return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)) } // ReleaseResource requires the complete capability AND positive cleanup evidence. // Mismatches and repeated releases are harmless, including after transfer. func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error { s.resourceMu.Lock() defer s.resourceMu.Unlock() return s.releaseResourceLocked(ctx, lease) } func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() ownerResource := lease.Resource if lease.pool != "" { if lease.Target == nil || !ValidResourceKey(lease.Target.Member) { return fmt.Errorf("invalid pooled resource lease") } ownerResource = lease.Target.Member } lease.Resource = ownerResource n, err := deleteResourceOwner(ctx, tx, lease) if err != nil { return err } if n == 0 { return nil } next, err := s.handoffResourceTx(ctx, tx, ownerResource) if err != nil { return err } if err := tx.Commit(); err != nil { return err } if next.token != "" { s.resourceWaiters[next.ResourceVisit] <- next } return nil } // handoffResourceTx is shared by ordinary release and inspected recovery. It // validates the in-memory waiter before changing durable ownership so a stale // server cannot grant a capability to a runner that no longer exists. func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) { var next ResourceLease next.Resource = resource rows, err := tx.QueryContext(ctx, `SELECT queue_order,resource_key,pool_name,run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? OR pool_name<>'' ORDER BY queue_order`, resource) if err != nil { return ResourceLease{}, err } defer rows.Close() var selected *protocol.ResourceTarget var queue int64 var waiterResource string for rows.Next() { var candidate ResourceLease var candidatePool string if err := rows.Scan(&queue, &waiterResource, &candidatePool, &candidate.RunID, &candidate.Stage, &candidate.Sequence, &candidate.token); err != nil { return ResourceLease{}, err } if candidatePool != "" { candidate.Resource = candidatePool candidate.pool = candidatePool } else { candidate.Resource = resource } if candidatePool == "" { if waiterResource != resource { continue } selected = &protocol.ResourceTarget{Member: resource} } else { members, e := s.resourcePool(ctx, tx, candidatePool) if e != nil { continue } found := false for _, member := range members { if !member.Enabled || member.Quarantined { continue } var count int if e = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); e != nil { return ResourceLease{}, e } if count == 0 { selected, found = poolTarget(member), true break } } if !found { continue } } if s.resourceWaiters[candidate.ResourceVisit] == nil { if candidatePool == "" { return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff") } continue } next = candidate next.Target = selected break } if err := rows.Err(); err != nil { return ResourceLease{}, err } if next.token == "" { return ResourceLease{}, nil } ownerResource := next.Resource if next.pool != "" { ownerResource = next.Target.Member } if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, ownerResource, next.RunID, next.Stage, next.Sequence, next.token, next.pool); err != nil { return ResourceLease{}, err } if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE queue_order=?`, queue); err != nil { return ResourceLease{}, err } if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { return ResourceLease{}, err } if next.pool == "" { next.Resource = ownerResource } return next, nil } func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) { result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token) if err != nil { return 0, err } return result.RowsAffected() } // recoverResourceOwners runs only at startup, after pending waiters have been // interrupted and removed. Every reclamation uses the persisted capability; // absent cleanup evidence is never replaced with a timestamp-based expiry. func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error { rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1") if err != nil { return err } var leases []ResourceLease for rows.Next() { var lease ResourceLease if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil { rows.Close() return err } leases = append(leases, lease) } err = rows.Err() rows.Close() if err != nil { return err } for _, lease := range leases { if _, err := deleteResourceOwner(ctx, tx, lease); err != nil { return err } } _, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'") return err }