b79d0401f3
Add local operator inspection and reconciliation for quarantined resource leases, including atomic audit-backed FIFO handoff and recovery protocol support.\n\nPreserve capability tokens in the authoritative store and reject non-loopback or ordinary client transport for recovery mutations.\n\nChecked packaged templates and packaged authoring skill references; none need updating. The task-owned docs note records shared README and skill reference follow-up for docs-fold.
459 lines
17 KiB
Go
459 lines
17 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
const maxRecoveryText = 512
|
|
|
|
func boundedRecoveryText(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
return value
|
|
}
|
|
|
|
// InspectResource exposes only bounded state; capability tokens never leave this
|
|
// package or enter a protocol value.
|
|
func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) {
|
|
if !ValidResourceKey(resource) {
|
|
return protocol.ResourceInspection{}, fmt.Errorf("invalid resource")
|
|
}
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}}
|
|
var runID int64
|
|
var stage string
|
|
var seq, cleanup int
|
|
err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup)
|
|
if err == nil {
|
|
var active int
|
|
if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
|
|
return out, err
|
|
}
|
|
out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0}
|
|
} else if err != sql.ErrNoRows {
|
|
return out, err
|
|
}
|
|
rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
for rows.Next() {
|
|
var w protocol.ResourceWaiter
|
|
if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil {
|
|
rows.Close()
|
|
return out, err
|
|
}
|
|
out.Waiters = append(out.Waiters, w)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return out, err
|
|
}
|
|
rows.Close()
|
|
audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
defer audits.Close()
|
|
for audits.Next() {
|
|
var a protocol.ResourceRecoveryAudit
|
|
a.Resource = resource
|
|
if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil {
|
|
audits.Close()
|
|
return out, err
|
|
}
|
|
out.Audits = append(out.Audits, a)
|
|
}
|
|
return out, audits.Err()
|
|
}
|
|
|
|
// ReconcileResource is the sole operator recovery mutation. The persisted
|
|
// capability is compared only inside this authoritative store operation.
|
|
func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error {
|
|
if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) {
|
|
return fmt.Errorf("invalid resource recovery identity")
|
|
}
|
|
if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) {
|
|
return fmt.Errorf("positive cleanup/restoration evidence is required")
|
|
}
|
|
if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText {
|
|
return fmt.Errorf("bounded actor is required")
|
|
}
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var token, stage string
|
|
var runID int64
|
|
var seq, cleanup int
|
|
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup)
|
|
if err == sql.ErrNoRows {
|
|
var count int
|
|
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil {
|
|
return err
|
|
}
|
|
if count > 0 {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("resource owner not found")
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if runID != request.RunID || stage != request.Stage || seq != request.Sequence {
|
|
return fmt.Errorf("resource owner visit is stale")
|
|
}
|
|
if cleanup != 0 {
|
|
return fmt.Errorf("resource owner is not quarantined")
|
|
}
|
|
var status string
|
|
if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil {
|
|
return err
|
|
}
|
|
if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning {
|
|
return fmt.Errorf("owning run is still active")
|
|
}
|
|
var active int
|
|
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
|
|
return err
|
|
}
|
|
if active != 0 {
|
|
return fmt.Errorf("owning agent is still active")
|
|
}
|
|
evidence := boundedRecoveryText(request.Evidence.Summary)
|
|
if request.Evidence.NoMutatingAction {
|
|
evidence = boundedRecoveryText("no mutating action occurred; " + evidence)
|
|
}
|
|
evidenceHash := sha256.Sum256([]byte(evidence))
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil {
|
|
return err
|
|
}
|
|
owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token}
|
|
if _, err := deleteResourceOwner(ctx, tx, owner); err != nil {
|
|
return err
|
|
}
|
|
next, err := s.handoffResourceTx(ctx, tx, request.Resource)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if next.token != "" {
|
|
s.resourceWaiters[next.ResourceVisit] <- next
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validRecoveryEvidence(value string) bool {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) {
|
|
return false
|
|
}
|
|
if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") {
|
|
return false
|
|
}
|
|
if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
|
|
return false
|
|
}
|
|
for _, r := range value {
|
|
if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
|
|
type ResourceVisit struct {
|
|
Resource string
|
|
RunID int64
|
|
Stage string
|
|
Sequence int
|
|
}
|
|
|
|
// ResourceLease is an opaque capability. Its credential is deliberately not exported
|
|
// or serializable, and must never be put in model input or public wire metadata.
|
|
type ResourceLease struct {
|
|
ResourceVisit
|
|
token string
|
|
}
|
|
|
|
// RequestResource acquires immediately or registers one FIFO waiter/channel. The
|
|
// mutex covers only short DB transactions and channel bookkeeping, never execution.
|
|
func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit")
|
|
}
|
|
if ch := s.resourceWaiters[visit]; ch != nil {
|
|
wait, err := s.resourceWait(ctx, visit.RunID)
|
|
return ResourceLease{}, ch, wait, err
|
|
}
|
|
credential := make([]byte, 32)
|
|
if _, err := rand.Read(credential); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
var status string
|
|
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
|
|
}
|
|
var existing ResourceLease
|
|
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token)
|
|
if err == nil {
|
|
if existing.ResourceVisit != visit {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
|
|
}
|
|
return existing, nil, nil, nil
|
|
}
|
|
if err != sql.ErrNoRows {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token)
|
|
VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if n == 1 {
|
|
return lease, nil, nil, tx.Commit()
|
|
}
|
|
wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
|
|
encoded, err := json.Marshal(wait)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if s.resourceWaiters == nil {
|
|
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
|
|
}
|
|
ch := make(chan ResourceLease, 1)
|
|
s.resourceWaiters[visit] = ch
|
|
return ResourceLease{}, ch, wait, nil
|
|
}
|
|
|
|
func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) {
|
|
var wait protocol.ResourceWait
|
|
err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince)
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
return &wait, err
|
|
}
|
|
|
|
// CancelResource removes a queued visit, or returns a concurrently reserved grant
|
|
// to the queue before any execution is allowed. It cannot cancel another visit.
|
|
func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
defer delete(s.resourceWaiters, visit)
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := deleted.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n > 0 {
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
ch := s.resourceWaiters[visit]
|
|
// A grant is buffered while this registry entry exists. Cancellation can
|
|
// reclaim only that not-yet-accepted capability, never an executing owner.
|
|
var grant ResourceLease
|
|
if ch != nil {
|
|
select {
|
|
case grant = <-ch:
|
|
default:
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if grant.token != "" {
|
|
if err := s.SetResourceCleanup(ctx, grant, true); err != nil {
|
|
return err
|
|
}
|
|
return s.releaseResourceLocked(ctx, grant)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AcceptResource removes the channel exactly once. Cancellation is rechecked by
|
|
// the runner before BeginStep; a canceled accepted grant is released as unused.
|
|
func (s *Store) AcceptResource(lease ResourceLease) {
|
|
s.resourceMu.Lock()
|
|
delete(s.resourceWaiters, lease.ResourceVisit)
|
|
s.resourceMu.Unlock()
|
|
}
|
|
|
|
// HasResourceOwner is used only for orphan/restart reconciliation, never for a
|
|
// resource-free stage visit.
|
|
func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) {
|
|
var exists bool
|
|
err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists)
|
|
return exists, err
|
|
}
|
|
|
|
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
|
|
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
|
|
return err
|
|
}
|
|
|
|
// SetResourceCleanup records positive quiescence durably before release. False
|
|
// quarantines the owner; neither age nor another run can remove the lease.
|
|
func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error {
|
|
diagnostic := ""
|
|
if !confirmed {
|
|
diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation"
|
|
}
|
|
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
|
|
}
|
|
|
|
// ReleaseResource requires the complete capability AND positive cleanup evidence.
|
|
// Mismatches and repeated releases are harmless, including after transfer.
|
|
func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
return s.releaseResourceLocked(ctx, lease)
|
|
}
|
|
|
|
func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
n, err := deleteResourceOwner(ctx, tx, lease)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n == 0 {
|
|
return nil
|
|
}
|
|
next, err := s.handoffResourceTx(ctx, tx, lease.Resource)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if next.token != "" {
|
|
s.resourceWaiters[next.ResourceVisit] <- next
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// handoffResourceTx is shared by ordinary release and inspected recovery. It
|
|
// validates the in-memory waiter before changing durable ownership so a stale
|
|
// server cannot grant a capability to a runner that no longer exists.
|
|
func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) {
|
|
var next ResourceLease
|
|
next.Resource = resource
|
|
err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
|
|
if err == sql.ErrNoRows {
|
|
return ResourceLease{}, nil
|
|
}
|
|
if err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
if s.resourceWaiters[next.ResourceVisit] == nil {
|
|
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
|
|
return ResourceLease{}, err
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
|
|
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return result.RowsAffected()
|
|
}
|
|
|
|
// recoverResourceOwners runs only at startup, after pending waiters have been
|
|
// interrupted and removed. Every reclamation uses the persisted capability;
|
|
// absent cleanup evidence is never replaced with a timestamp-based expiry.
|
|
func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error {
|
|
rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var leases []ResourceLease
|
|
for rows.Next() {
|
|
var lease ResourceLease
|
|
if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
leases = append(leases, lease)
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, lease := range leases {
|
|
if _, err := deleteResourceOwner(ctx, tx, lease); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'")
|
|
return err
|
|
}
|