9130ae8c7a
Add durable FIFO capacity-one resource leases for executable stages, including cleanup-confirmed release, conservative restart quarantine, queue state events, and CLI/TUI/Android waiting presentation. Add scheduler, cleanup, protocol, client, and Android coverage plus a proposed Android v3 resource draft. Checked packaged templates and authoring skill references: neither needs updating in this item. The per-item docs note records the shared documentation that docs-fold must update; no shared README or packaged reference was edited.
278 lines
10 KiB
Go
278 lines
10 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
|
|
type ResourceVisit struct {
|
|
Resource string
|
|
RunID int64
|
|
Stage string
|
|
Sequence int
|
|
}
|
|
|
|
// ResourceLease is an opaque capability. Its credential is deliberately not exported
|
|
// or serializable, and must never be put in model input or public wire metadata.
|
|
type ResourceLease struct {
|
|
ResourceVisit
|
|
token string
|
|
}
|
|
|
|
// RequestResource acquires immediately or registers one FIFO waiter/channel. The
|
|
// mutex covers only short DB transactions and channel bookkeeping, never execution.
|
|
func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
if !ValidResourceKey(visit.Resource) || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource visit")
|
|
}
|
|
if ch := s.resourceWaiters[visit]; ch != nil {
|
|
wait, err := s.resourceWait(ctx, visit.RunID)
|
|
return ResourceLease{}, ch, wait, err
|
|
}
|
|
credential := make([]byte, 32)
|
|
if _, err := rand.Read(credential); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
var status string
|
|
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
|
|
}
|
|
var existing ResourceLease
|
|
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token)
|
|
if err == nil {
|
|
if existing.ResourceVisit != visit {
|
|
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
|
|
}
|
|
return existing, nil, nil, nil
|
|
}
|
|
if err != sql.ErrNoRows {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token)
|
|
VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if n == 1 {
|
|
return lease, nil, nil, tx.Commit()
|
|
}
|
|
wait := &protocol.ResourceWait{Resource: visit.Resource, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
|
|
encoded, err := json.Marshal(wait)
|
|
if err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?)`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return ResourceLease{}, nil, nil, err
|
|
}
|
|
if s.resourceWaiters == nil {
|
|
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
|
|
}
|
|
ch := make(chan ResourceLease, 1)
|
|
s.resourceWaiters[visit] = ch
|
|
return ResourceLease{}, ch, wait, nil
|
|
}
|
|
|
|
func (s *Store) resourceWait(ctx context.Context, id int64) (*protocol.ResourceWait, error) {
|
|
var wait protocol.ResourceWait
|
|
err := s.db.QueryRowContext(ctx, "SELECT resource_key,stage_key,sequence,queued_at FROM resource_waiters WHERE run_id=?", id).Scan(&wait.Resource, &wait.Stage, &wait.Sequence, &wait.QueuedSince)
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
return &wait, err
|
|
}
|
|
|
|
// CancelResource removes a queued visit, or returns a concurrently reserved grant
|
|
// to the queue before any execution is allowed. It cannot cancel another visit.
|
|
func (s *Store) CancelResource(ctx context.Context, visit ResourceVisit) error {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
defer delete(s.resourceWaiters, visit)
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
deleted, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=?", visit.Resource, visit.RunID, visit.Stage, visit.Sequence)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := deleted.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n > 0 {
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", visit.RunID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
ch := s.resourceWaiters[visit]
|
|
// A grant is buffered while this registry entry exists. Cancellation can
|
|
// reclaim only that not-yet-accepted capability, never an executing owner.
|
|
var grant ResourceLease
|
|
if ch != nil {
|
|
select {
|
|
case grant = <-ch:
|
|
default:
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if grant.token != "" {
|
|
if err := s.SetResourceCleanup(ctx, grant, true); err != nil {
|
|
return err
|
|
}
|
|
return s.releaseResourceLocked(ctx, grant)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AcceptResource removes the channel exactly once. Cancellation is rechecked by
|
|
// the runner before BeginStep; a canceled accepted grant is released as unused.
|
|
func (s *Store) AcceptResource(lease ResourceLease) {
|
|
s.resourceMu.Lock()
|
|
delete(s.resourceWaiters, lease.ResourceVisit)
|
|
s.resourceMu.Unlock()
|
|
}
|
|
|
|
// HasResourceOwner is used only for orphan/restart reconciliation, never for a
|
|
// resource-free stage visit.
|
|
func (s *Store) HasResourceOwner(ctx context.Context, runID int64) (bool, error) {
|
|
var exists bool
|
|
err := s.db.QueryRowContext(ctx, "SELECT EXISTS(SELECT 1 FROM resource_owners WHERE run_id=?)", runID).Scan(&exists)
|
|
return exists, err
|
|
}
|
|
|
|
func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) error {
|
|
_, err := s.db.ExecContext(ctx, "UPDATE run_steps SET agent_active=0 WHERE run_id=? AND session_id=?", owner.RunID, owner.ID)
|
|
return err
|
|
}
|
|
|
|
// SetResourceCleanup records positive quiescence durably before release. False
|
|
// quarantines the owner; neither age nor another run can remove the lease.
|
|
func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error {
|
|
diagnostic := ""
|
|
if !confirmed {
|
|
diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation"
|
|
}
|
|
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
|
|
}
|
|
|
|
// ReleaseResource requires the complete capability AND positive cleanup evidence.
|
|
// Mismatches and repeated releases are harmless, including after transfer.
|
|
func (s *Store) ReleaseResource(ctx context.Context, lease ResourceLease) error {
|
|
s.resourceMu.Lock()
|
|
defer s.resourceMu.Unlock()
|
|
return s.releaseResourceLocked(ctx, lease)
|
|
}
|
|
|
|
func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
n, err := deleteResourceOwner(ctx, tx, lease)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n == 0 {
|
|
return nil
|
|
}
|
|
var next ResourceLease
|
|
next.Resource = lease.Resource
|
|
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
|
|
if err != nil && err != sql.ErrNoRows {
|
|
return err
|
|
}
|
|
if err == nil {
|
|
if s.resourceWaiters[next.ResourceVisit] == nil {
|
|
return fmt.Errorf("resource waiter has no live runner; refusing handoff")
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if next.token != "" {
|
|
s.resourceWaiters[next.ResourceVisit] <- next
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
|
|
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return result.RowsAffected()
|
|
}
|
|
|
|
// recoverResourceOwners runs only at startup, after pending waiters have been
|
|
// interrupted and removed. Every reclamation uses the persisted capability;
|
|
// absent cleanup evidence is never replaced with a timestamp-based expiry.
|
|
func recoverResourceOwners(ctx context.Context, tx *sql.Tx) error {
|
|
rows, err := tx.QueryContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE cleanup_confirmed=1")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var leases []ResourceLease
|
|
for rows.Next() {
|
|
var lease ResourceLease
|
|
if err := rows.Scan(&lease.Resource, &lease.RunID, &lease.Stage, &lease.Sequence, &lease.token); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
leases = append(leases, lease)
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, lease := range leases {
|
|
if _, err := deleteResourceOwner(ctx, tx, lease); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err = tx.ExecContext(ctx, "UPDATE resource_owners SET diagnostic='Server stopped; resource quarantined pending inspected cleanup/restoration'")
|
|
return err
|
|
}
|