feat: recover inspected quarantined resources

Add local operator inspection and reconciliation for quarantined resource leases, including atomic audit-backed FIFO handoff and recovery protocol support.\n\nPreserve capability tokens in the authoritative store and reject non-loopback or ordinary client transport for recovery mutations.\n\nChecked packaged templates and packaged authoring skill references; none need updating. The task-owned docs note records shared README and skill reference follow-up for docs-fold.
This commit is contained in:
2026-10-18 19:24:32 +01:00
parent 27ad7daecd
commit b79d0401f3
8 changed files with 444 additions and 71 deletions
+19
View File
@@ -0,0 +1,19 @@
# Inspected resource recovery
The operator recovery contract now keeps quarantined resource ownership and its
audit transactionally coupled to FIFO handoff. Inspection is redacted and
reconciliation is accepted only over a loopback operator connection with the
exact owner visit plus positive cleanup/restoration evidence. The persisted
capability remains private to the authoritative live store; missing waiter
channels fail closed without changing owner or audit state.
Use the configured `PIPELINES_URL` endpoint, not a disposable URL. First run
the redacted resource inspection, then reconcile only after the parked visit
has been inspected and quiesced, supplying its exact run/stage/sequence and
explicit evidence (including `no mutating action occurred` where applicable).
The old deployed server does not support this action; deploy a compatible
server before attempting recovery. This is an operator trust boundary, not an
OS sandbox or unrestricted authorization grant.
The shared README and packaged OpenCode skill references should be updated by
the manual `docs-fold` process; they are intentionally not edited here.
+76 -3
View File
@@ -7,6 +7,7 @@ import (
"flag"
"fmt"
"io"
"net/url"
"os"
"os/exec"
"path/filepath"
@@ -36,7 +37,7 @@ type App struct {
type options struct {
harness string
url, name, prompt, title, template, workspace, paths, key, opencode, model string
option, feedback string
option, feedback, evidence, actor string
output string
conflict string
kind string
@@ -45,7 +46,9 @@ type options struct {
keepSet bool
all bool
json, source, noOpen, yes, allow, createOnly, force, wait bool
version, expected, limit int
version, expected, limit, recoverySequence int
recoveryRun int64
recoveryStage, resource string
timeout, waitTimeout time.Duration
stageModels stageModelFlags
}
@@ -177,6 +180,13 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
case "decide":
f.StringVar(&o.option, "option", "", "Option key to choose among the waiting stage's edges (required)")
f.StringVar(&o.feedback, "feedback", "", "Optional free-text feedback recorded with the decision")
case "resource-inspect":
f.StringVar(&o.resource, "resource", "", "Exclusive resource key")
case "resource-reconcile":
f.StringVar(&o.evidence, "evidence", "", "Bounded positive cleanup/restoration evidence (required)")
f.StringVar(&o.actor, "actor", "operator", "Bounded operator identity")
f.BoolVar(&o.allow, "no-mutating-action", false, "Evidence explicitly records that no mutating action occurred")
f.BoolVar(&o.yes, "yes", false, "Confirm reconciliation without prompting")
case "debug":
f.StringVar(&o.harness, "harness", "", "Debugging harness: configured opencode/pi, or nvim")
f.BoolVar(&o.yes, "yes", false, "Approve opening a debugging session without prompting")
@@ -462,8 +472,16 @@ func (a App) Run(ctx context.Context, args []string) error {
if requestTimeout <= 0 {
requestTimeout = 30 * time.Second
}
connectURL := o.url
if command == "resource-inspect" || command == "resource-reconcile" {
var transportErr error
connectURL, transportErr = operatorTransportURL(o.url)
if transportErr != nil {
return transportErr
}
}
dialCtx, cancel := context.WithTimeout(ctx, requestTimeout)
c, err := client.Open(dialCtx, o.url)
c, err := client.Open(dialCtx, connectURL)
cancel()
if err != nil {
if command == "status" && len(pos) == 0 {
@@ -897,6 +915,51 @@ func (a App) Run(ctx context.Context, args []string) error {
}
}
return fmt.Errorf("template %q not found", key)
case "resource-inspect":
if len(pos) > 1 {
return fmt.Errorf("resource-inspect takes one resource key")
}
if o.resource == "" && len(pos) == 1 {
o.resource = pos[0]
}
if strings.TrimSpace(o.resource) == "" {
return fmt.Errorf("resource-inspect requires a resource key")
}
r, err := call(protocol.Message{Type: protocol.InspectResource, Resource: o.resource})
if err != nil {
return err
}
if r.ResourceInspection == nil {
return fmt.Errorf("server did not return resource inspection")
}
return a.output(r.ResourceInspection)
case "resource-reconcile":
if len(pos) != 4 {
return fmt.Errorf("resource-reconcile requires resource, run-id, stage and sequence")
}
runID, err := strconv.ParseInt(pos[1], 10, 64)
if err != nil || runID <= 0 {
return fmt.Errorf("run ID must be positive")
}
sequence, err := strconv.Atoi(pos[3])
if err != nil || sequence <= 0 {
return fmt.Errorf("sequence must be positive")
}
if strings.TrimSpace(o.evidence) == "" {
return fmt.Errorf("--evidence is required")
}
if err := a.confirmAction(fmt.Sprintf("Reconcile quarantined resource %s for run #%d?", pos[0], runID), o.yes); err != nil {
return err
}
r, err := call(protocol.Message{Type: protocol.ReconcileResource, Resource: pos[0], ResourceRecovery: &protocol.ResourceRecoveryRequest{Resource: pos[0], RunID: runID, Stage: pos[2], Sequence: sequence, Actor: o.actor, Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: o.allow, Summary: o.evidence}}})
if err != nil {
return err
}
if o.json {
return a.output(r)
}
_, err = fmt.Fprintf(a.Out, "Reconciled resource %s for run #%d\n", pos[0], runID)
return err
case "inspect", "cancel", "retry":
value, err := one()
if err != nil {
@@ -1039,6 +1102,16 @@ func (a App) Run(ctx context.Context, args []string) error {
return nil
}
func operatorTransportURL(raw string) (string, error) {
u, err := url.Parse(raw)
if err != nil || u.Scheme == "" || u.Host == "" {
return "", fmt.Errorf("invalid configured pipeline URL")
}
u.Path = "/operator-connect"
u.RawPath = ""
return u.String(), nil
}
func (a App) output(value any) error {
encoder := json.NewEncoder(a.Out)
encoder.SetIndent("", " ")
+35 -33
View File
@@ -2,39 +2,41 @@ package cli
// Help is also the public command inventory for agents and parity checks.
var usage = map[string]string{
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
"drafts": "drafts [--json]",
"draft-delete": "draft-delete <draft.md> [--json]",
"validate": "validate <draft.md> [--json]",
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
"list": "list [--json]",
"search": "search QUERY [--type KIND] [--limit N] [--json]",
"show": "show <name|id> [--version N] [--json]",
"versions": "versions <name|id> [--json]",
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
"cancel": "cancel <run-id> [--json]",
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
"title": "title <run-id> \"<title>\" [--json]",
"debug": "debug <run-id> [--json | --no-open]",
"chat-adopt": "chat-adopt <session-id> [--json]",
"connection": "connection [--json]",
"ping": "ping [--json]",
"version": "version [--json]",
"status": "status [--json]",
"stats": "stats [--json]",
"rpc": "rpc <request.json|->",
"retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]",
"tui": "tui",
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
"drafts": "drafts [--json]",
"draft-delete": "draft-delete <draft.md> [--json]",
"validate": "validate <draft.md> [--json]",
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
"list": "list [--json]",
"search": "search QUERY [--type KIND] [--limit N] [--json]",
"show": "show <name|id> [--version N] [--json]",
"versions": "versions <name|id> [--json]",
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
"cancel": "cancel <run-id> [--json]",
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
"title": "title <run-id> \"<title>\" [--json]",
"debug": "debug <run-id> [--json | --no-open]",
"chat-adopt": "chat-adopt <session-id> [--json]",
"connection": "connection [--json]",
"ping": "ping [--json]",
"version": "version [--json]",
"status": "status [--json]",
"stats": "stats [--json]",
"resource-inspect": "resource-inspect <resource> [--json]",
"resource-reconcile": "resource-reconcile <resource> <run-id> <stage> <sequence> --evidence TEXT [--no-mutating-action] [--yes] [--json]",
"rpc": "rpc <request.json|->",
"retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]",
"tui": "tui",
}
var notes = map[string]string{
+199 -18
View File
@@ -3,15 +3,184 @@ package pipeline
import (
"context"
"crypto/rand"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
const maxRecoveryText = 512
func boundedRecoveryText(value string) string {
value = strings.TrimSpace(value)
return value
}
// InspectResource exposes only bounded state; capability tokens never leave this
// package or enter a protocol value.
func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) {
if !ValidResourceKey(resource) {
return protocol.ResourceInspection{}, fmt.Errorf("invalid resource")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}}
var runID int64
var stage string
var seq, cleanup int
err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup)
if err == nil {
var active int
if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
return out, err
}
out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0}
} else if err != sql.ErrNoRows {
return out, err
}
rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource)
if err != nil {
return out, err
}
for rows.Next() {
var w protocol.ResourceWaiter
if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil {
rows.Close()
return out, err
}
out.Waiters = append(out.Waiters, w)
}
if err := rows.Err(); err != nil {
rows.Close()
return out, err
}
rows.Close()
audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource)
if err != nil {
return out, err
}
defer audits.Close()
for audits.Next() {
var a protocol.ResourceRecoveryAudit
a.Resource = resource
if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil {
audits.Close()
return out, err
}
out.Audits = append(out.Audits, a)
}
return out, audits.Err()
}
// ReconcileResource is the sole operator recovery mutation. The persisted
// capability is compared only inside this authoritative store operation.
func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error {
if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) {
return fmt.Errorf("invalid resource recovery identity")
}
if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) {
return fmt.Errorf("positive cleanup/restoration evidence is required")
}
if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText {
return fmt.Errorf("bounded actor is required")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
var token, stage string
var runID int64
var seq, cleanup int
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup)
if err == sql.ErrNoRows {
var count int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil {
return err
}
if count > 0 {
return nil
}
return fmt.Errorf("resource owner not found")
}
if err != nil {
return err
}
if runID != request.RunID || stage != request.Stage || seq != request.Sequence {
return fmt.Errorf("resource owner visit is stale")
}
if cleanup != 0 {
return fmt.Errorf("resource owner is not quarantined")
}
var status string
if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil {
return err
}
if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning {
return fmt.Errorf("owning run is still active")
}
var active int
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
return err
}
if active != 0 {
return fmt.Errorf("owning agent is still active")
}
evidence := boundedRecoveryText(request.Evidence.Summary)
if request.Evidence.NoMutatingAction {
evidence = boundedRecoveryText("no mutating action occurred; " + evidence)
}
evidenceHash := sha256.Sum256([]byte(evidence))
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil {
return err
}
owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token}
if _, err := deleteResourceOwner(ctx, tx, owner); err != nil {
return err
}
next, err := s.handoffResourceTx(ctx, tx, request.Resource)
if err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
if next.token != "" {
s.resourceWaiters[next.ResourceVisit] <- next
}
return nil
}
func validRecoveryEvidence(value string) bool {
value = strings.TrimSpace(value)
if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) {
return false
}
if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") {
return false
}
if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
return false
}
for _, r := range value {
if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') {
return false
}
}
return true
}
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
type ResourceVisit struct {
Resource string
@@ -208,26 +377,10 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
if n == 0 {
return nil
}
var next ResourceLease
next.Resource = lease.Resource
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
if err != nil && err != sql.ErrNoRows {
next, err := s.handoffResourceTx(ctx, tx, lease.Resource)
if err != nil {
return err
}
if err == nil {
if s.resourceWaiters[next.ResourceVisit] == nil {
return fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
@@ -237,6 +390,34 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
return nil
}
// handoffResourceTx is shared by ordinary release and inspected recovery. It
// validates the in-memory waiter before changing durable ownership so a stale
// server cannot grant a capability to a runner that no longer exists.
func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) {
var next ResourceLease
next.Resource = resource
err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
if err == sql.ErrNoRows {
return ResourceLease{}, nil
}
if err != nil {
return ResourceLease{}, err
}
if s.resourceWaiters[next.ResourceVisit] == nil {
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
return ResourceLease{}, err
}
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil {
return ResourceLease{}, err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return ResourceLease{}, err
}
return next, nil
}
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
if err != nil {
+6
View File
@@ -73,6 +73,12 @@ CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,q
if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_recovery_audit (
resource_key TEXT NOT NULL, run_id INTEGER NOT NULL, stage_key TEXT NOT NULL,
sequence INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL,
evidence TEXT NOT NULL, result TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)`); err != nil {
return err
}
return s.migrateRunModels()
}
+22 -15
View File
@@ -3,19 +3,23 @@ package protocol
import "git.a-wan.net/alan/pipelines/internal/buildinfo"
const (
Ping = "ping"
Pong = "pong"
Error = "error"
Submit = "submit"
Submitted = "submitted"
List = "list_pipelines"
PipelineList = "pipelines"
GetPipeline = "get_pipeline"
Pipeline = "pipeline"
ListTemplates = "list_templates"
Templates = "templates"
Validate = "validate"
Validated = "validated"
Ping = "ping"
Pong = "pong"
Error = "error"
Submit = "submit"
Submitted = "submitted"
List = "list_pipelines"
PipelineList = "pipelines"
GetPipeline = "get_pipeline"
Pipeline = "pipeline"
ListTemplates = "list_templates"
Templates = "templates"
Validate = "validate"
Validated = "validated"
InspectResource = "inspect_resource"
ResourceInspectionResult = "resource_inspection"
ReconcileResource = "reconcile_resource"
ResourceReconciled = "resource_reconciled"
)
type Stage struct {
@@ -191,6 +195,9 @@ type Message struct {
// RunEvent is the server-pushed run state delta (see runevents.go) and
// RunIDs the optional subscribe scope, so a client asks for the runs it
// displays. Both are optional so older peers ignore them.
RunEvent *RunEventFrame `json:"run_event,omitempty"`
RunIDs []int64 `json:"run_ids,omitempty"`
RunEvent *RunEventFrame `json:"run_event,omitempty"`
RunIDs []int64 `json:"run_ids,omitempty"`
Resource string `json:"resource,omitempty"`
ResourceInspection *ResourceInspection `json:"resource_inspection,omitempty"`
ResourceRecovery *ResourceRecoveryRequest `json:"resource_recovery,omitempty"`
}
+46
View File
@@ -0,0 +1,46 @@
package protocol
type ResourceInspection struct {
Resource string `json:"resource"`
Owner *ResourceOwner `json:"owner,omitempty"`
Waiters []ResourceWaiter `json:"waiters,omitempty"`
Audits []ResourceRecoveryAudit `json:"audits,omitempty"`
}
type ResourceOwner struct {
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
CleanupConfirmed bool `json:"cleanup_confirmed"`
Quarantined bool `json:"quarantined"`
AgentActive bool `json:"agent_active"`
}
type ResourceWaiter struct {
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
QueuedAt string `json:"queued_at"`
}
type ResourceRecoveryAudit struct {
Resource string `json:"resource"`
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
Actor string `json:"actor"`
Source string `json:"source"`
Evidence string `json:"evidence"`
Result string `json:"result"`
CreatedAt string `json:"created_at"`
}
type ResourceRecoveryEvidence struct {
CleanupConfirmed bool `json:"cleanup_confirmed"`
NoMutatingAction bool `json:"no_mutating_action"`
Summary string `json:"summary"`
}
type ResourceRecoveryRequest struct {
Resource string `json:"resource"`
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
Evidence *ResourceRecoveryEvidence `json:"evidence,omitempty"`
Actor string `json:"actor"`
}
+41 -2
View File
@@ -2,7 +2,9 @@ package server
import (
"context"
"fmt"
"log"
"net"
"net/http"
"strings"
"sync"
@@ -119,7 +121,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
_, _ = w.Write([]byte(`{"healthy":true}`))
})
mux.HandleFunc("GET /connect", func(w http.ResponseWriter, r *http.Request) {
connect := func(w http.ResponseWriter, r *http.Request) {
conn, err := websocket.Accept(w, r, nil)
if err != nil {
return
@@ -268,6 +270,32 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
response.Error = err.Error()
}
}
} else if (request.Type == protocol.InspectResource || request.Type == protocol.ReconcileResource) && len(stores) > 0 {
var err error
response.Type = protocol.ResourceInspectionResult
// Recovery has a separate transport path. A stage/model client may
// also be local, so loopback alone is not an authority boundary.
// The CLI uses /operator-connect; the ordinary /connect path is
// deliberately refused even from loopback.
if r.URL.Path != "/operator-connect" || !isLoopbackPeer(r.RemoteAddr) {
response.Type = protocol.Error
response.Error = "resource recovery requires the local operator transport"
} else if request.Type == protocol.InspectResource {
var inspection protocol.ResourceInspection
inspection, err = stores[0].InspectResource(ctx, request.Resource)
response.ResourceInspection = &inspection
} else {
response.Type = protocol.ResourceReconciled
if request.ResourceRecovery == nil {
err = fmt.Errorf("resource recovery evidence is required")
} else {
err = stores[0].ReconcileResource(ctx, *request.ResourceRecovery)
}
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence) {
var err error
switch request.Type {
@@ -323,10 +351,21 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion)
}
})
}
mux.HandleFunc("GET /connect", connect)
mux.HandleFunc("GET /operator-connect", connect)
return &runHandler{Handler: mux, runner: runs}
}
func isLoopbackPeer(remoteAddr string) bool {
host, _, err := net.SplitHostPort(remoteAddr)
if err != nil {
return false
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
type runHandler struct {
http.Handler
runner *runner