Files
pipelines/internal/pipeline/resources.go
T
alan 9130ae8c7a Add stage-exclusive resource waiting
Add durable FIFO capacity-one resource leases for executable stages, including cleanup-confirmed release, conservative restart quarantine, queue state events, and CLI/TUI/Android waiting presentation. Add scheduler, cleanup, protocol, client, and Android coverage plus a proposed Android v3 resource draft.

Checked packaged templates and authoring skill references: neither needs updating in this item. The per-item docs note records the shared documentation that docs-fold must update; no shared README or packaged reference was edited.
2026-10-18 12:30:30 +01:00

278 lines
10 KiB
Go

package pipeline
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
type ResourceVisit struct {
Resource string
RunID int64
Stage string
Sequence int
}
// ResourceLease is an opaque capability. Its credential is deliberately not exported
// or serializable, and must never be put in model input or public wire metadata.
type ResourceLease struct {
ResourceVisit
token string
}
// RequestResource acquires immediately or registers one FIFO waiter/channel. The
// mutex covers only short DB transactions and channel bookkeeping, never execution.
func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit")
}
if ch := s.resourceWaiters[visit]; ch != nil {
wait, err := s.resourceWait(ctx, visit.RunID)
return ResourceLease{}, ch, wait, err
}
credential := make([]byte, 32)
if _, err := rand.Read(credential); err != nil {
return ResourceLease{}, nil, nil, err
}
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return ResourceLease{}, nil, nil, err
}
defer tx.Rollback()
var status string
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
return ResourceLease{}, nil, nil, err
}
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
}
var existing ResourceLease
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token)
if err == nil {
if existing.ResourceVisit != visit {
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
}
return existing, nil, nil, nil
}
if err != sql.ErrNoRows {
return ResourceLease{}, nil, nil, err
}
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token)
VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token)
if err != nil {
return ResourceLease{}, nil, nil, err
}
n, err := result.RowsAffected()
if err != nil {
return ResourceLease{}, nil, nil, err
}
if n == 1 {
return lease, nil, nil, tx.Commit()
}
wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
encoded, err := json.Marshal(wait)
if err != nil {
return ResourceLease{}, nil, nil, err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
return ResourceLease{}, nil, nil, err
}
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
return ResourceLease{}, nil, nil, err
}
if err := tx.Commit(); err != nil {
return ResourceLease{}, nil, nil, err
}
if s.resourceWaiters == nil {
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
}
ch := make(chan ResourceLease, 1)
s.resourceWaiters[visit] = ch
return ResourceLease{}, ch, wait, nil
}
func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) {
var wait protocol.ResourceWait
err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince)
if err == sql.ErrNoRows {
return nil, nil
}
return &wait, err
}
// CancelResource removes a queued visit, or returns a concurrently reserved grant
// to the queue before any execution is allowed. It cannot cancel another visit.
func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
defer delete(s.resourceWaiters, visit)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence)
if err != nil {
return err
}
n, err := deleted.RowsAffected()
if err != nil {
return err
}
if n > 0 {
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil {
return err
}
}
ch := s.resourceWaiters[visit]
// A grant is buffered while this registry entry exists. Cancellation can
// reclaim only that not-yet-accepted capability, never an executing owner.
var grant ResourceLease
if ch != nil {
select {
case grant = <-ch:
default:
}
}
if err := tx.Commit(); err != nil {
return err
}
if grant.token != "" {
if err := s.SetResourceCleanup(ctx, grant, true); err != nil {
return err
}
return s.releaseResourceLocked(ctx, grant)
}
return nil
}
// AcceptResource removes the channel exactly once. Cancellation is rechecked by
// the runner before BeginStep; a canceled accepted grant is released as unused.
func (s *Store) AcceptResource(lease ResourceLease) {
s.resourceMu.Lock()
delete(s.resourceWaiters, lease.ResourceVisit)
s.resourceMu.Unlock()
}
// HasResourceOwner is used only for orphan/restart reconciliation, never for a
// resource-free stage visit.
func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) {
var exists bool
err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists)
return exists, err
}
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
return err
}
// SetResourceCleanup records positive quiescence durably before release. False
// quarantines the owner; neither age nor another run can remove the lease.
func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error {
diagnostic := ""
if !confirmed {
diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation"
}
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
}
// ReleaseResource requires the complete capability AND positive cleanup evidence.
// Mismatches and repeated releases are harmless, including after transfer.
func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error {
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
return s.releaseResourceLocked(ctx, lease)
}
func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
n, err := deleteResourceOwner(ctx, tx, lease)
if err != nil {
return err
}
if n == 0 {
return nil
}
var next ResourceLease
next.Resource = lease.Resource
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
if err != nil && err != sql.ErrNoRows {
return err
}
if err == nil {
if s.resourceWaiters[next.ResourceVisit] == nil {
return fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
if next.token != "" {
s.resourceWaiters[next.ResourceVisit] <- next
}
return nil
}
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
// recoverResourceOwners runs only at startup, after pending waiters have been
// interrupted and removed. Every reclamation uses the persisted capability;
// absent cleanup evidence is never replaced with a timestamp-based expiry.
func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error {
rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1")
if err != nil {
return err
}
var leases []ResourceLease
for rows.Next() {
var lease ResourceLease
if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil {
rows.Close()
return err
}
leases = append(leases, lease)
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
for _, lease := range leases {
if _, err := deleteResourceOwner(ctx, tx, lease); err != nil {
return err
}
}
_, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'")
return err
}