feat: recover inspected quarantined resources
Add local operator inspection and reconciliation for quarantined resource leases, including atomic audit-backed FIFO handoff and recovery protocol support.\n\nPreserve capability tokens in the authoritative store and reject non-loopback or ordinary client transport for recovery mutations.\n\nChecked packaged templates and packaged authoring skill references; none need updating. The task-owned docs note records shared README and skill reference follow-up for docs-fold.
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
# Inspected resource recovery
|
||||
|
||||
The operator recovery contract now keeps quarantined resource ownership and its
|
||||
audit transactionally coupled to FIFO handoff. Inspection is redacted and
|
||||
reconciliation is accepted only over a loopback operator connection with the
|
||||
exact owner visit plus positive cleanup/restoration evidence. The persisted
|
||||
capability remains private to the authoritative live store; missing waiter
|
||||
channels fail closed without changing owner or audit state.
|
||||
|
||||
Use the configured `PIPELINES_URL` endpoint, not a disposable URL. First run
|
||||
the redacted resource inspection, then reconcile only after the parked visit
|
||||
has been inspected and quiesced, supplying its exact run/stage/sequence and
|
||||
explicit evidence (including `no mutating action occurred` where applicable).
|
||||
The old deployed server does not support this action; deploy a compatible
|
||||
server before attempting recovery. This is an operator trust boundary, not an
|
||||
OS sandbox or unrestricted authorization grant.
|
||||
|
||||
The shared README and packaged OpenCode skill references should be updated by
|
||||
the manual `docs-fold` process; they are intentionally not edited here.
|
||||
+76
-3
@@ -7,6 +7,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -36,7 +37,7 @@ type App struct {
|
||||
type options struct {
|
||||
harness string
|
||||
url, name, prompt, title, template, workspace, paths, key, opencode, model string
|
||||
option, feedback string
|
||||
option, feedback, evidence, actor string
|
||||
output string
|
||||
conflict string
|
||||
kind string
|
||||
@@ -45,7 +46,9 @@ type options struct {
|
||||
keepSet bool
|
||||
all bool
|
||||
json, source, noOpen, yes, allow, createOnly, force, wait bool
|
||||
version, expected, limit int
|
||||
version, expected, limit, recoverySequence int
|
||||
recoveryRun int64
|
||||
recoveryStage, resource string
|
||||
timeout, waitTimeout time.Duration
|
||||
stageModels stageModelFlags
|
||||
}
|
||||
@@ -177,6 +180,13 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
case "decide":
|
||||
f.StringVar(&o.option, "option", "", "Option key to choose among the waiting stage's edges (required)")
|
||||
f.StringVar(&o.feedback, "feedback", "", "Optional free-text feedback recorded with the decision")
|
||||
case "resource-inspect":
|
||||
f.StringVar(&o.resource, "resource", "", "Exclusive resource key")
|
||||
case "resource-reconcile":
|
||||
f.StringVar(&o.evidence, "evidence", "", "Bounded positive cleanup/restoration evidence (required)")
|
||||
f.StringVar(&o.actor, "actor", "operator", "Bounded operator identity")
|
||||
f.BoolVar(&o.allow, "no-mutating-action", false, "Evidence explicitly records that no mutating action occurred")
|
||||
f.BoolVar(&o.yes, "yes", false, "Confirm reconciliation without prompting")
|
||||
case "debug":
|
||||
f.StringVar(&o.harness, "harness", "", "Debugging harness: configured opencode/pi, or nvim")
|
||||
f.BoolVar(&o.yes, "yes", false, "Approve opening a debugging session without prompting")
|
||||
@@ -462,8 +472,16 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
if requestTimeout <= 0 {
|
||||
requestTimeout = 30 * time.Second
|
||||
}
|
||||
connectURL := o.url
|
||||
if command == "resource-inspect" || command == "resource-reconcile" {
|
||||
var transportErr error
|
||||
connectURL, transportErr = operatorTransportURL(o.url)
|
||||
if transportErr != nil {
|
||||
return transportErr
|
||||
}
|
||||
}
|
||||
dialCtx, cancel := context.WithTimeout(ctx, requestTimeout)
|
||||
c, err := client.Open(dialCtx, o.url)
|
||||
c, err := client.Open(dialCtx, connectURL)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if command == "status" && len(pos) == 0 {
|
||||
@@ -897,6 +915,51 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("template %q not found", key)
|
||||
case "resource-inspect":
|
||||
if len(pos) > 1 {
|
||||
return fmt.Errorf("resource-inspect takes one resource key")
|
||||
}
|
||||
if o.resource == "" && len(pos) == 1 {
|
||||
o.resource = pos[0]
|
||||
}
|
||||
if strings.TrimSpace(o.resource) == "" {
|
||||
return fmt.Errorf("resource-inspect requires a resource key")
|
||||
}
|
||||
r, err := call(protocol.Message{Type: protocol.InspectResource, Resource: o.resource})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.ResourceInspection == nil {
|
||||
return fmt.Errorf("server did not return resource inspection")
|
||||
}
|
||||
return a.output(r.ResourceInspection)
|
||||
case "resource-reconcile":
|
||||
if len(pos) != 4 {
|
||||
return fmt.Errorf("resource-reconcile requires resource, run-id, stage and sequence")
|
||||
}
|
||||
runID, err := strconv.ParseInt(pos[1], 10, 64)
|
||||
if err != nil || runID <= 0 {
|
||||
return fmt.Errorf("run ID must be positive")
|
||||
}
|
||||
sequence, err := strconv.Atoi(pos[3])
|
||||
if err != nil || sequence <= 0 {
|
||||
return fmt.Errorf("sequence must be positive")
|
||||
}
|
||||
if strings.TrimSpace(o.evidence) == "" {
|
||||
return fmt.Errorf("--evidence is required")
|
||||
}
|
||||
if err := a.confirmAction(fmt.Sprintf("Reconcile quarantined resource %s for run #%d?", pos[0], runID), o.yes); err != nil {
|
||||
return err
|
||||
}
|
||||
r, err := call(protocol.Message{Type: protocol.ReconcileResource, Resource: pos[0], ResourceRecovery: &protocol.ResourceRecoveryRequest{Resource: pos[0], RunID: runID, Stage: pos[2], Sequence: sequence, Actor: o.actor, Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: o.allow, Summary: o.evidence}}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if o.json {
|
||||
return a.output(r)
|
||||
}
|
||||
_, err = fmt.Fprintf(a.Out, "Reconciled resource %s for run #%d\n", pos[0], runID)
|
||||
return err
|
||||
case "inspect", "cancel", "retry":
|
||||
value, err := one()
|
||||
if err != nil {
|
||||
@@ -1039,6 +1102,16 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func operatorTransportURL(raw string) (string, error) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return "", fmt.Errorf("invalid configured pipeline URL")
|
||||
}
|
||||
u.Path = "/operator-connect"
|
||||
u.RawPath = ""
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (a App) output(value any) error {
|
||||
encoder := json.NewEncoder(a.Out)
|
||||
encoder.SetIndent("", " ")
|
||||
|
||||
+35
-33
@@ -2,39 +2,41 @@ package cli
|
||||
|
||||
// Help is also the public command inventory for agents and parity checks.
|
||||
var usage = map[string]string{
|
||||
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
|
||||
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
|
||||
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
|
||||
"drafts": "drafts [--json]",
|
||||
"draft-delete": "draft-delete <draft.md> [--json]",
|
||||
"validate": "validate <draft.md> [--json]",
|
||||
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
|
||||
"list": "list [--json]",
|
||||
"search": "search QUERY [--type KIND] [--limit N] [--json]",
|
||||
"show": "show <name|id> [--version N] [--json]",
|
||||
"versions": "versions <name|id> [--json]",
|
||||
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
|
||||
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
|
||||
"templates": "templates [--json]",
|
||||
"template": "template <key> [--source | --json]",
|
||||
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"runs": "runs [--pipeline NAME] [--json]",
|
||||
"inspect": "inspect <run-id> [--json]",
|
||||
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
|
||||
"cancel": "cancel <run-id> [--json]",
|
||||
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
|
||||
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"title": "title <run-id> \"<title>\" [--json]",
|
||||
"debug": "debug <run-id> [--json | --no-open]",
|
||||
"chat-adopt": "chat-adopt <session-id> [--json]",
|
||||
"connection": "connection [--json]",
|
||||
"ping": "ping [--json]",
|
||||
"version": "version [--json]",
|
||||
"status": "status [--json]",
|
||||
"stats": "stats [--json]",
|
||||
"rpc": "rpc <request.json|->",
|
||||
"retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]",
|
||||
"tui": "tui",
|
||||
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
|
||||
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
|
||||
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
|
||||
"drafts": "drafts [--json]",
|
||||
"draft-delete": "draft-delete <draft.md> [--json]",
|
||||
"validate": "validate <draft.md> [--json]",
|
||||
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
|
||||
"list": "list [--json]",
|
||||
"search": "search QUERY [--type KIND] [--limit N] [--json]",
|
||||
"show": "show <name|id> [--version N] [--json]",
|
||||
"versions": "versions <name|id> [--json]",
|
||||
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
|
||||
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
|
||||
"templates": "templates [--json]",
|
||||
"template": "template <key> [--source | --json]",
|
||||
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"runs": "runs [--pipeline NAME] [--json]",
|
||||
"inspect": "inspect <run-id> [--json]",
|
||||
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
|
||||
"cancel": "cancel <run-id> [--json]",
|
||||
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
|
||||
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"title": "title <run-id> \"<title>\" [--json]",
|
||||
"debug": "debug <run-id> [--json | --no-open]",
|
||||
"chat-adopt": "chat-adopt <session-id> [--json]",
|
||||
"connection": "connection [--json]",
|
||||
"ping": "ping [--json]",
|
||||
"version": "version [--json]",
|
||||
"status": "status [--json]",
|
||||
"stats": "stats [--json]",
|
||||
"resource-inspect": "resource-inspect <resource> [--json]",
|
||||
"resource-reconcile": "resource-reconcile <resource> <run-id> <stage> <sequence> --evidence TEXT [--no-mutating-action] [--yes] [--json]",
|
||||
"rpc": "rpc <request.json|->",
|
||||
"retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]",
|
||||
"tui": "tui",
|
||||
}
|
||||
|
||||
var notes = map[string]string{
|
||||
|
||||
+199
-18
@@ -3,15 +3,184 @@ package pipeline
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
const maxRecoveryText = 512
|
||||
|
||||
func boundedRecoveryText(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
return value
|
||||
}
|
||||
|
||||
// InspectResource exposes only bounded state; capability tokens never leave this
|
||||
// package or enter a protocol value.
|
||||
func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) {
|
||||
if !ValidResourceKey(resource) {
|
||||
return protocol.ResourceInspection{}, fmt.Errorf("invalid resource")
|
||||
}
|
||||
s.resourceMu.Lock()
|
||||
defer s.resourceMu.Unlock()
|
||||
out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}}
|
||||
var runID int64
|
||||
var stage string
|
||||
var seq, cleanup int
|
||||
err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup)
|
||||
if err == nil {
|
||||
var active int
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0}
|
||||
} else if err != sql.ErrNoRows {
|
||||
return out, err
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var w protocol.ResourceWaiter
|
||||
if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil {
|
||||
rows.Close()
|
||||
return out, err
|
||||
}
|
||||
out.Waiters = append(out.Waiters, w)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return out, err
|
||||
}
|
||||
rows.Close()
|
||||
audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
defer audits.Close()
|
||||
for audits.Next() {
|
||||
var a protocol.ResourceRecoveryAudit
|
||||
a.Resource = resource
|
||||
if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil {
|
||||
audits.Close()
|
||||
return out, err
|
||||
}
|
||||
out.Audits = append(out.Audits, a)
|
||||
}
|
||||
return out, audits.Err()
|
||||
}
|
||||
|
||||
// ReconcileResource is the sole operator recovery mutation. The persisted
|
||||
// capability is compared only inside this authoritative store operation.
|
||||
func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error {
|
||||
if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) {
|
||||
return fmt.Errorf("invalid resource recovery identity")
|
||||
}
|
||||
if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) {
|
||||
return fmt.Errorf("positive cleanup/restoration evidence is required")
|
||||
}
|
||||
if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText {
|
||||
return fmt.Errorf("bounded actor is required")
|
||||
}
|
||||
s.resourceMu.Lock()
|
||||
defer s.resourceMu.Unlock()
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var token, stage string
|
||||
var runID int64
|
||||
var seq, cleanup int
|
||||
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup)
|
||||
if err == sql.ErrNoRows {
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count > 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("resource owner not found")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if runID != request.RunID || stage != request.Stage || seq != request.Sequence {
|
||||
return fmt.Errorf("resource owner visit is stale")
|
||||
}
|
||||
if cleanup != 0 {
|
||||
return fmt.Errorf("resource owner is not quarantined")
|
||||
}
|
||||
var status string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil {
|
||||
return err
|
||||
}
|
||||
if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning {
|
||||
return fmt.Errorf("owning run is still active")
|
||||
}
|
||||
var active int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
|
||||
return err
|
||||
}
|
||||
if active != 0 {
|
||||
return fmt.Errorf("owning agent is still active")
|
||||
}
|
||||
evidence := boundedRecoveryText(request.Evidence.Summary)
|
||||
if request.Evidence.NoMutatingAction {
|
||||
evidence = boundedRecoveryText("no mutating action occurred; " + evidence)
|
||||
}
|
||||
evidenceHash := sha256.Sum256([]byte(evidence))
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil {
|
||||
return err
|
||||
}
|
||||
owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token}
|
||||
if _, err := deleteResourceOwner(ctx, tx, owner); err != nil {
|
||||
return err
|
||||
}
|
||||
next, err := s.handoffResourceTx(ctx, tx, request.Resource)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
if next.token != "" {
|
||||
s.resourceWaiters[next.ResourceVisit] <- next
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validRecoveryEvidence(value string) bool {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
|
||||
type ResourceVisit struct {
|
||||
Resource string
|
||||
@@ -208,26 +377,10 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
var next ResourceLease
|
||||
next.Resource = lease.Resource
|
||||
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
next, err := s.handoffResourceTx(ctx, tx, lease.Resource)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
if s.resourceWaiters[next.ResourceVisit] == nil {
|
||||
return fmt.Errorf("resource waiter has no live runner; refusing handoff")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -237,6 +390,34 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handoffResourceTx is shared by ordinary release and inspected recovery. It
|
||||
// validates the in-memory waiter before changing durable ownership so a stale
|
||||
// server cannot grant a capability to a runner that no longer exists.
|
||||
func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) {
|
||||
var next ResourceLease
|
||||
next.Resource = resource
|
||||
err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
|
||||
if err == sql.ErrNoRows {
|
||||
return ResourceLease{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if s.resourceWaiters[next.ResourceVisit] == nil {
|
||||
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
|
||||
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
|
||||
if err != nil {
|
||||
|
||||
@@ -73,6 +73,12 @@ CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,q
|
||||
if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_recovery_audit (
|
||||
resource_key TEXT NOT NULL, run_id INTEGER NOT NULL, stage_key TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL,
|
||||
evidence TEXT NOT NULL, result TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)`); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.migrateRunModels()
|
||||
}
|
||||
|
||||
|
||||
@@ -3,19 +3,23 @@ package protocol
|
||||
import "git.a-wan.net/alan/pipelines/internal/buildinfo"
|
||||
|
||||
const (
|
||||
Ping = "ping"
|
||||
Pong = "pong"
|
||||
Error = "error"
|
||||
Submit = "submit"
|
||||
Submitted = "submitted"
|
||||
List = "list_pipelines"
|
||||
PipelineList = "pipelines"
|
||||
GetPipeline = "get_pipeline"
|
||||
Pipeline = "pipeline"
|
||||
ListTemplates = "list_templates"
|
||||
Templates = "templates"
|
||||
Validate = "validate"
|
||||
Validated = "validated"
|
||||
Ping = "ping"
|
||||
Pong = "pong"
|
||||
Error = "error"
|
||||
Submit = "submit"
|
||||
Submitted = "submitted"
|
||||
List = "list_pipelines"
|
||||
PipelineList = "pipelines"
|
||||
GetPipeline = "get_pipeline"
|
||||
Pipeline = "pipeline"
|
||||
ListTemplates = "list_templates"
|
||||
Templates = "templates"
|
||||
Validate = "validate"
|
||||
Validated = "validated"
|
||||
InspectResource = "inspect_resource"
|
||||
ResourceInspectionResult = "resource_inspection"
|
||||
ReconcileResource = "reconcile_resource"
|
||||
ResourceReconciled = "resource_reconciled"
|
||||
)
|
||||
|
||||
type Stage struct {
|
||||
@@ -191,6 +195,9 @@ type Message struct {
|
||||
// RunEvent is the server-pushed run state delta (see runevents.go) and
|
||||
// RunIDs the optional subscribe scope, so a client asks for the runs it
|
||||
// displays. Both are optional so older peers ignore them.
|
||||
RunEvent *RunEventFrame `json:"run_event,omitempty"`
|
||||
RunIDs []int64 `json:"run_ids,omitempty"`
|
||||
RunEvent *RunEventFrame `json:"run_event,omitempty"`
|
||||
RunIDs []int64 `json:"run_ids,omitempty"`
|
||||
Resource string `json:"resource,omitempty"`
|
||||
ResourceInspection *ResourceInspection `json:"resource_inspection,omitempty"`
|
||||
ResourceRecovery *ResourceRecoveryRequest `json:"resource_recovery,omitempty"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package protocol
|
||||
|
||||
type ResourceInspection struct {
|
||||
Resource string `json:"resource"`
|
||||
Owner *ResourceOwner `json:"owner,omitempty"`
|
||||
Waiters []ResourceWaiter `json:"waiters,omitempty"`
|
||||
Audits []ResourceRecoveryAudit `json:"audits,omitempty"`
|
||||
}
|
||||
type ResourceOwner struct {
|
||||
RunID int64 `json:"run_id"`
|
||||
Stage string `json:"stage"`
|
||||
Sequence int `json:"sequence"`
|
||||
CleanupConfirmed bool `json:"cleanup_confirmed"`
|
||||
Quarantined bool `json:"quarantined"`
|
||||
AgentActive bool `json:"agent_active"`
|
||||
}
|
||||
type ResourceWaiter struct {
|
||||
RunID int64 `json:"run_id"`
|
||||
Stage string `json:"stage"`
|
||||
Sequence int `json:"sequence"`
|
||||
QueuedAt string `json:"queued_at"`
|
||||
}
|
||||
type ResourceRecoveryAudit struct {
|
||||
Resource string `json:"resource"`
|
||||
RunID int64 `json:"run_id"`
|
||||
Stage string `json:"stage"`
|
||||
Sequence int `json:"sequence"`
|
||||
Actor string `json:"actor"`
|
||||
Source string `json:"source"`
|
||||
Evidence string `json:"evidence"`
|
||||
Result string `json:"result"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
type ResourceRecoveryEvidence struct {
|
||||
CleanupConfirmed bool `json:"cleanup_confirmed"`
|
||||
NoMutatingAction bool `json:"no_mutating_action"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
type ResourceRecoveryRequest struct {
|
||||
Resource string `json:"resource"`
|
||||
RunID int64 `json:"run_id"`
|
||||
Stage string `json:"stage"`
|
||||
Sequence int `json:"sequence"`
|
||||
Evidence *ResourceRecoveryEvidence `json:"evidence,omitempty"`
|
||||
Actor string `json:"actor"`
|
||||
}
|
||||
@@ -2,7 +2,9 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -119,7 +121,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"healthy":true}`))
|
||||
})
|
||||
mux.HandleFunc("GET /connect", func(w http.ResponseWriter, r *http.Request) {
|
||||
connect := func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := websocket.Accept(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -268,6 +270,32 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
response.Error = err.Error()
|
||||
}
|
||||
}
|
||||
} else if (request.Type == protocol.InspectResource || request.Type == protocol.ReconcileResource) && len(stores) > 0 {
|
||||
var err error
|
||||
response.Type = protocol.ResourceInspectionResult
|
||||
// Recovery has a separate transport path. A stage/model client may
|
||||
// also be local, so loopback alone is not an authority boundary.
|
||||
// The CLI uses /operator-connect; the ordinary /connect path is
|
||||
// deliberately refused even from loopback.
|
||||
if r.URL.Path != "/operator-connect" || !isLoopbackPeer(r.RemoteAddr) {
|
||||
response.Type = protocol.Error
|
||||
response.Error = "resource recovery requires the local operator transport"
|
||||
} else if request.Type == protocol.InspectResource {
|
||||
var inspection protocol.ResourceInspection
|
||||
inspection, err = stores[0].InspectResource(ctx, request.Resource)
|
||||
response.ResourceInspection = &inspection
|
||||
} else {
|
||||
response.Type = protocol.ResourceReconciled
|
||||
if request.ResourceRecovery == nil {
|
||||
err = fmt.Errorf("resource recovery evidence is required")
|
||||
} else {
|
||||
err = stores[0].ReconcileResource(ctx, *request.ResourceRecovery)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
response.Type = protocol.Error
|
||||
response.Error = err.Error()
|
||||
}
|
||||
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence) {
|
||||
var err error
|
||||
switch request.Type {
|
||||
@@ -323,10 +351,21 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
}
|
||||
log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion)
|
||||
}
|
||||
})
|
||||
}
|
||||
mux.HandleFunc("GET /connect", connect)
|
||||
mux.HandleFunc("GET /operator-connect", connect)
|
||||
return &runHandler{Handler: mux, runner: runs}
|
||||
}
|
||||
|
||||
func isLoopbackPeer(remoteAddr string) bool {
|
||||
host, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
|
||||
type runHandler struct {
|
||||
http.Handler
|
||||
runner *runner
|
||||
|
||||
Reference in New Issue
Block a user