Merge branch 'main' into background-task-output-api

This commit is contained in:
2026-10-18 20:23:48 +01:00
10 changed files with 941 additions and 71 deletions
+19
View File
@@ -0,0 +1,19 @@
# Inspected resource recovery
The operator recovery contract now keeps quarantined resource ownership and its
audit transactionally coupled to FIFO handoff. Inspection is redacted and
reconciliation is accepted only over a loopback operator connection with the
exact owner visit plus positive cleanup/restoration evidence. The persisted
capability remains private to the authoritative live store; missing waiter
channels fail closed without changing owner or audit state.
Use the configured `PIPELINES_URL` endpoint, not a disposable URL. First run
the redacted resource inspection, then reconcile only after the parked visit
has been inspected and quiesced, supplying its exact run/stage/sequence and
explicit evidence (including `no mutating action occurred` where applicable).
The old deployed server does not support this action; deploy a compatible
server before attempting recovery. This is an operator trust boundary, not an
OS sandbox or unrestricted authorization grant.
The shared README and packaged OpenCode skill references should be updated by
the manual `docs-fold` process; they are intentionally not edited here.
+76 -3
View File
@@ -7,6 +7,7 @@ import (
"flag"
"fmt"
"io"
"net/url"
"os"
"os/exec"
"path/filepath"
@@ -36,7 +37,7 @@ type App struct {
type options struct {
harness string
url, name, prompt, title, template, workspace, paths, key, opencode, model string
option, feedback string
option, feedback, evidence, actor string
output string
conflict string
kind string
@@ -45,7 +46,9 @@ type options struct {
keepSet bool
all bool
json, source, noOpen, yes, allow, createOnly, force, wait bool
version, expected, limit int
version, expected, limit, recoverySequence int
recoveryRun int64
recoveryStage, resource string
timeout, waitTimeout time.Duration
stageModels stageModelFlags
}
@@ -177,6 +180,13 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
case "decide":
f.StringVar(&o.option, "option", "", "Option key to choose among the waiting stage's edges (required)")
f.StringVar(&o.feedback, "feedback", "", "Optional free-text feedback recorded with the decision")
case "resource-inspect":
f.StringVar(&o.resource, "resource", "", "Exclusive resource key")
case "resource-reconcile":
f.StringVar(&o.evidence, "evidence", "", "Bounded positive cleanup/restoration evidence (required)")
f.StringVar(&o.actor, "actor", "operator", "Bounded operator identity")
f.BoolVar(&o.allow, "no-mutating-action", false, "Evidence explicitly records that no mutating action occurred")
f.BoolVar(&o.yes, "yes", false, "Confirm reconciliation without prompting")
case "debug":
f.StringVar(&o.harness, "harness", "", "Debugging harness: configured opencode/pi, or nvim")
f.BoolVar(&o.yes, "yes", false, "Approve opening a debugging session without prompting")
@@ -462,8 +472,16 @@ func (a App) Run(ctx context.Context, args []string) error {
if requestTimeout <= 0 {
requestTimeout = 30 * time.Second
}
connectURL := o.url
if command == "resource-inspect" || command == "resource-reconcile" {
var transportErr error
connectURL, transportErr = operatorTransportURL(o.url)
if transportErr != nil {
return transportErr
}
}
dialCtx, cancel := context.WithTimeout(ctx, requestTimeout)
c, err := client.Open(dialCtx, o.url)
c, err := client.Open(dialCtx, connectURL)
cancel()
if err != nil {
if command == "status" && len(pos) == 0 {
@@ -897,6 +915,51 @@ func (a App) Run(ctx context.Context, args []string) error {
}
}
return fmt.Errorf("template %q not found", key)
case "resource-inspect":
if len(pos) > 1 {
return fmt.Errorf("resource-inspect takes one resource key")
}
if o.resource == "" && len(pos) == 1 {
o.resource = pos[0]
}
if strings.TrimSpace(o.resource) == "" {
return fmt.Errorf("resource-inspect requires a resource key")
}
r, err := call(protocol.Message{Type: protocol.InspectResource, Resource: o.resource})
if err != nil {
return err
}
if r.ResourceInspection == nil {
return fmt.Errorf("server did not return resource inspection")
}
return a.output(r.ResourceInspection)
case "resource-reconcile":
if len(pos) != 4 {
return fmt.Errorf("resource-reconcile requires resource, run-id, stage and sequence")
}
runID, err := strconv.ParseInt(pos[1], 10, 64)
if err != nil || runID <= 0 {
return fmt.Errorf("run ID must be positive")
}
sequence, err := strconv.Atoi(pos[3])
if err != nil || sequence <= 0 {
return fmt.Errorf("sequence must be positive")
}
if strings.TrimSpace(o.evidence) == "" {
return fmt.Errorf("--evidence is required")
}
if err := a.confirmAction(fmt.Sprintf("Reconcile quarantined resource %s for run #%d?", pos[0], runID), o.yes); err != nil {
return err
}
r, err := call(protocol.Message{Type: protocol.ReconcileResource, Resource: pos[0], ResourceRecovery: &protocol.ResourceRecoveryRequest{Resource: pos[0], RunID: runID, Stage: pos[2], Sequence: sequence, Actor: o.actor, Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: o.allow, Summary: o.evidence}}})
if err != nil {
return err
}
if o.json {
return a.output(r)
}
_, err = fmt.Fprintf(a.Out, "Reconciled resource %s for run #%d\n", pos[0], runID)
return err
case "inspect", "cancel", "retry":
value, err := one()
if err != nil {
@@ -1039,6 +1102,16 @@ func (a App) Run(ctx context.Context, args []string) error {
return nil
}
func operatorTransportURL(raw string) (string, error) {
u, err := url.Parse(raw)
if err != nil || u.Scheme == "" || u.Host == "" {
return "", fmt.Errorf("invalid configured pipeline URL")
}
u.Path = "/operator-connect"
u.RawPath = ""
return u.String(), nil
}
func (a App) output(value any) error {
encoder := json.NewEncoder(a.Out)
encoder.SetIndent("", " ")
+35 -33
View File
@@ -2,39 +2,41 @@ package cli
// Help is also the public command inventory for agents and parity checks.
var usage = map[string]string{
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
"drafts": "drafts [--json]",
"draft-delete": "draft-delete <draft.md> [--json]",
"validate": "validate <draft.md> [--json]",
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
"list": "list [--json]",
"search": "search QUERY [--type KIND] [--limit N] [--json]",
"show": "show <name|id> [--version N] [--json]",
"versions": "versions <name|id> [--json]",
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
"cancel": "cancel <run-id> [--json]",
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
"title": "title <run-id> \"<title>\" [--json]",
"debug": "debug <run-id> [--json | --no-open]",
"chat-adopt": "chat-adopt <session-id> [--json]",
"connection": "connection [--json]",
"ping": "ping [--json]",
"version": "version [--json]",
"status": "status [--json]",
"stats": "stats [--json]",
"rpc": "rpc <request.json|->",
"retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]",
"tui": "tui",
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
"drafts": "drafts [--json]",
"draft-delete": "draft-delete <draft.md> [--json]",
"validate": "validate <draft.md> [--json]",
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
"list": "list [--json]",
"search": "search QUERY [--type KIND] [--limit N] [--json]",
"show": "show <name|id> [--version N] [--json]",
"versions": "versions <name|id> [--json]",
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
"cancel": "cancel <run-id> [--json]",
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
"title": "title <run-id> \"<title>\" [--json]",
"debug": "debug <run-id> [--json | --no-open]",
"chat-adopt": "chat-adopt <session-id> [--json]",
"connection": "connection [--json]",
"ping": "ping [--json]",
"version": "version [--json]",
"status": "status [--json]",
"stats": "stats [--json]",
"resource-inspect": "resource-inspect <resource> [--json]",
"resource-reconcile": "resource-reconcile <resource> <run-id> <stage> <sequence> --evidence TEXT [--no-mutating-action] [--yes] [--json]",
"rpc": "rpc <request.json|->",
"retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]",
"tui": "tui",
}
var notes = map[string]string{
+199 -18
View File
@@ -3,15 +3,184 @@ package pipeline
import (
"context"
"crypto/rand"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
const maxRecoveryText = 512
func boundedRecoveryText(value string) string {
value = strings.TrimSpace(value)
return value
}
// InspectResource exposes only bounded state; capability tokens never leave this
// package or enter a protocol value.
func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) {
if !ValidResourceKey(resource) {
return protocol.ResourceInspection{}, fmt.Errorf("invalid resource")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}}
var runID int64
var stage string
var seq, cleanup int
err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup)
if err == nil {
var active int
if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
return out, err
}
out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0}
} else if err != sql.ErrNoRows {
return out, err
}
rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource)
if err != nil {
return out, err
}
for rows.Next() {
var w protocol.ResourceWaiter
if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil {
rows.Close()
return out, err
}
out.Waiters = append(out.Waiters, w)
}
if err := rows.Err(); err != nil {
rows.Close()
return out, err
}
rows.Close()
audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource)
if err != nil {
return out, err
}
defer audits.Close()
for audits.Next() {
var a protocol.ResourceRecoveryAudit
a.Resource = resource
if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil {
audits.Close()
return out, err
}
out.Audits = append(out.Audits, a)
}
return out, audits.Err()
}
// ReconcileResource is the sole operator recovery mutation. The persisted
// capability is compared only inside this authoritative store operation.
func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error {
if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) {
return fmt.Errorf("invalid resource recovery identity")
}
if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) {
return fmt.Errorf("positive cleanup/restoration evidence is required")
}
if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText {
return fmt.Errorf("bounded actor is required")
}
s.resourceMu.Lock()
defer s.resourceMu.Unlock()
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
var token, stage string
var runID int64
var seq, cleanup int
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup)
if err == sql.ErrNoRows {
var count int
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil {
return err
}
if count > 0 {
return nil
}
return fmt.Errorf("resource owner not found")
}
if err != nil {
return err
}
if runID != request.RunID || stage != request.Stage || seq != request.Sequence {
return fmt.Errorf("resource owner visit is stale")
}
if cleanup != 0 {
return fmt.Errorf("resource owner is not quarantined")
}
var status string
if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil {
return err
}
if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning {
return fmt.Errorf("owning run is still active")
}
var active int
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil {
return err
}
if active != 0 {
return fmt.Errorf("owning agent is still active")
}
evidence := boundedRecoveryText(request.Evidence.Summary)
if request.Evidence.NoMutatingAction {
evidence = boundedRecoveryText("no mutating action occurred; " + evidence)
}
evidenceHash := sha256.Sum256([]byte(evidence))
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil {
return err
}
owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token}
if _, err := deleteResourceOwner(ctx, tx, owner); err != nil {
return err
}
next, err := s.handoffResourceTx(ctx, tx, request.Resource)
if err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
if next.token != "" {
s.resourceWaiters[next.ResourceVisit] <- next
}
return nil
}
func validRecoveryEvidence(value string) bool {
value = strings.TrimSpace(value)
if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) {
return false
}
if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") {
return false
}
if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
return false
}
for _, r := range value {
if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') {
return false
}
}
return true
}
// ResourceVisit identifies a pending visit, before it consumes execution accounting.
type ResourceVisit struct {
Resource string
@@ -208,26 +377,10 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
if n == 0 {
return nil
}
var next ResourceLease
next.Resource = lease.Resource
err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
if err != nil && err != sql.ErrNoRows {
next, err := s.handoffResourceTx(ctx, tx, lease.Resource)
if err != nil {
return err
}
if err == nil {
if s.resourceWaiters[next.ResourceVisit] == nil {
return fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
@@ -237,6 +390,34 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
return nil
}
// handoffResourceTx is shared by ordinary release and inspected recovery. It
// validates the in-memory waiter before changing durable ownership so a stale
// server cannot grant a capability to a runner that no longer exists.
func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) {
var next ResourceLease
next.Resource = resource
err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
if err == sql.ErrNoRows {
return ResourceLease{}, nil
}
if err != nil {
return ResourceLease{}, err
}
if s.resourceWaiters[next.ResourceVisit] == nil {
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
return ResourceLease{}, err
}
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil {
return ResourceLease{}, err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
return ResourceLease{}, err
}
return next, nil
}
func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) {
result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)
if err != nil {
+242
View File
@@ -9,6 +9,8 @@ import (
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestResourceDeclaration(t *testing.T) {
@@ -286,3 +288,243 @@ func TestResourceRestartQuarantineAndRetention(t *testing.T) {
})
}
}
func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
waiter := create("X")
if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil {
t.Fatalf("register waiter: %v", err)
}
// Simulate a runner disappearing while its durable waiter remains. The
// authoritative operation must reject the handoff before audit or cleanup.
delete(s.resourceWaiters, waiter)
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
})
if err == nil {
t.Fatal("reconciled without a live waiter channel")
}
var cleanup, audits, owners int
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
t.Fatal(err)
}
if cleanup != 0 || audits != 0 || owners != 1 {
t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners)
}
}
func TestResourceReconcileTransfersAndAudits(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
waiter := create("X")
_, ch, _, err := s.RequestResource(ctx, waiter)
if err != nil || ch == nil {
t.Fatalf("register waiter: %v", err)
}
if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
}); err != nil {
t.Fatal(err)
}
select {
case next := <-ch:
s.AcceptResource(next)
if next.ResourceVisit != waiter {
t.Fatalf("wrong handoff: %#v", next.ResourceVisit)
}
case <-time.After(time.Second):
t.Fatal("reconciliation did not wake waiter")
}
var audits int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if audits != 1 {
t.Fatalf("recovery audit count=%d", audits)
}
}
func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
successor := create("X")
_, ch, _, err := s.RequestResource(ctx, successor)
if err != nil || ch == nil {
t.Fatalf("register successor: %v", err)
}
request := protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{
CleanupConfirmed: true,
NoMutatingAction: true,
Summary: "inspection confirmed no mutating action occurred",
},
}
if err := s.ReconcileResource(ctx, request); err != nil {
t.Fatal(err)
}
var successorLease ResourceLease
select {
case successorLease = <-ch:
case <-time.After(time.Second):
t.Fatal("reconciliation did not hand off to successor")
}
if successorLease.ResourceVisit != successor {
t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit)
}
if successorLease.token == "" {
t.Fatal("successor capability was empty")
}
s.AcceptResource(successorLease)
// The old request is an idempotent replay after the handoff. It must not
// match or release the successor's private capability.
if err := s.ReconcileResource(ctx, request); err == nil {
t.Fatal("old recovery replay was accepted against successor owner")
}
inspection, err := s.InspectResource(ctx, owner.Resource)
if err != nil {
t.Fatal(err)
}
if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence {
t.Fatalf("successor was not preserved: %#v", inspection.Owner)
}
if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID {
t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits)
}
if len(s.resourceWaiters) != 0 {
t.Fatal("successor channel registry leaked")
}
var owners, waiters, audits int
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if owners != 1 || waiters != 0 || audits != 1 {
t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits)
}
}
func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) {
ctx := context.Background()
test := func(t *testing.T, setup func(*Store, ResourceLease) int) {
s, create := resourceStore(t)
owner := acquireTestResource(t, s, create("X"))
expectedCleanup := setup(s, owner)
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"},
})
if err == nil {
t.Fatal("reconciled unsafe owner")
}
var owners, audits, cleanup int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if owners != 1 || audits != 0 || cleanup != expectedCleanup {
t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
}
}
t.Run("active run", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil {
t.Fatal(err)
}
return 0
})
})
t.Run("active agent", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil {
t.Fatal(err)
}
return 0
})
})
t.Run("already confirmed", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
t.Fatal(err)
}
return 1
})
})
}
func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) {
ctx := context.Background()
for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} {
t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) {
s, create := resourceStore(t)
owner := acquireTestResource(t, s, create("X"))
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary},
})
if err == nil {
t.Fatal("accepted malformed evidence")
}
var owners, audits, cleanup int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if owners != 1 || audits != 0 || cleanup != 0 {
t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
}
})
}
}
+6
View File
@@ -73,6 +73,12 @@ CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,q
if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil {
return err
}
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_recovery_audit (
resource_key TEXT NOT NULL, run_id INTEGER NOT NULL, stage_key TEXT NOT NULL,
sequence INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL,
evidence TEXT NOT NULL, result TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)`); err != nil {
return err
}
return s.migrateRunModels()
}
+22 -15
View File
@@ -3,19 +3,23 @@ package protocol
import "git.a-wan.net/alan/pipelines/internal/buildinfo"
const (
Ping = "ping"
Pong = "pong"
Error = "error"
Submit = "submit"
Submitted = "submitted"
List = "list_pipelines"
PipelineList = "pipelines"
GetPipeline = "get_pipeline"
Pipeline = "pipeline"
ListTemplates = "list_templates"
Templates = "templates"
Validate = "validate"
Validated = "validated"
Ping = "ping"
Pong = "pong"
Error = "error"
Submit = "submit"
Submitted = "submitted"
List = "list_pipelines"
PipelineList = "pipelines"
GetPipeline = "get_pipeline"
Pipeline = "pipeline"
ListTemplates = "list_templates"
Templates = "templates"
Validate = "validate"
Validated = "validated"
InspectResource = "inspect_resource"
ResourceInspectionResult = "resource_inspection"
ReconcileResource = "reconcile_resource"
ResourceReconciled = "resource_reconciled"
)
type Stage struct {
@@ -194,6 +198,9 @@ type Message struct {
// RunEvent is the server-pushed run state delta (see runevents.go) and
// RunIDs the optional subscribe scope, so a client asks for the runs it
// displays. Both are optional so older peers ignore them.
RunEvent *RunEventFrame `json:"run_event,omitempty"`
RunIDs []int64 `json:"run_ids,omitempty"`
RunEvent *RunEventFrame `json:"run_event,omitempty"`
RunIDs []int64 `json:"run_ids,omitempty"`
Resource string `json:"resource,omitempty"`
ResourceInspection *ResourceInspection `json:"resource_inspection,omitempty"`
ResourceRecovery *ResourceRecoveryRequest `json:"resource_recovery,omitempty"`
}
+46
View File
@@ -0,0 +1,46 @@
package protocol
type ResourceInspection struct {
Resource string `json:"resource"`
Owner *ResourceOwner `json:"owner,omitempty"`
Waiters []ResourceWaiter `json:"waiters,omitempty"`
Audits []ResourceRecoveryAudit `json:"audits,omitempty"`
}
type ResourceOwner struct {
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
CleanupConfirmed bool `json:"cleanup_confirmed"`
Quarantined bool `json:"quarantined"`
AgentActive bool `json:"agent_active"`
}
type ResourceWaiter struct {
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
QueuedAt string `json:"queued_at"`
}
type ResourceRecoveryAudit struct {
Resource string `json:"resource"`
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
Actor string `json:"actor"`
Source string `json:"source"`
Evidence string `json:"evidence"`
Result string `json:"result"`
CreatedAt string `json:"created_at"`
}
type ResourceRecoveryEvidence struct {
CleanupConfirmed bool `json:"cleanup_confirmed"`
NoMutatingAction bool `json:"no_mutating_action"`
Summary string `json:"summary"`
}
type ResourceRecoveryRequest struct {
Resource string `json:"resource"`
RunID int64 `json:"run_id"`
Stage string `json:"stage"`
Sequence int `json:"sequence"`
Evidence *ResourceRecoveryEvidence `json:"evidence,omitempty"`
Actor string `json:"actor"`
}
+255
View File
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"encoding/json"
"fmt"
"net"
"net/http/httptest"
"os"
"path/filepath"
@@ -15,11 +16,30 @@ import (
"time"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/client"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
type fixedRemoteListener struct{ net.Listener }
func (l fixedRemoteListener) Accept() (net.Conn, error) {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return fixedRemoteConn{Conn: conn}, nil
}
type fixedRemoteConn struct{ net.Conn }
func (fixedRemoteConn) RemoteAddr() net.Addr {
return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443}
}
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
// executor. It never calls inference, a phone, or a deployed server.
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
@@ -110,6 +130,241 @@ func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
}
func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"
if _, err := store.Submit(ctx, "operator-resource", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) string {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
}
return out.String()
}
inspection := runCLI("resource-inspect", visit.Resource, "--json")
if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") {
t.Fatalf("inspection leaked capability: %s", inspection)
}
runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json")
inspection = runCLI("resource-inspect", visit.Resource, "--json")
var recovered protocol.ResourceInspection
if err := json.Unmarshal([]byte(inspection), &recovered); err != nil {
t.Fatal(err)
}
if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" {
t.Fatalf("CLI recovery state was not persisted: %s", inspection)
}
}
func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
lease, ch, _, err := store.RequestResource(ctx, visit)
if err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) error {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
err := app.Run(ctx, append([]string{"--url", endpoint}, args...))
if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") {
t.Fatalf("private capability appeared in CLI output: %s", out.String())
}
return err
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted wrong owner")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale stage")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale sequence")
}
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
t.Fatal(err)
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted non-quarantined owner")
}
inspection, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 {
t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits))
}
}
func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) {
for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} {
if isLoopbackPeer(remote) {
t.Fatalf("accepted non-loopback peer %q", remote)
}
}
for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} {
if !isLoopbackPeer(remote) {
t.Fatalf("rejected loopback peer %q", remote)
}
}
// X-Forwarded-For is deliberately not consulted: a proxy-supplied header
// cannot turn a remote WebSocket peer into the local operator transport.
}
func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
before, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
server := httptest.NewUnstartedServer(Handler(ctx, store))
server.Listener = fixedRemoteListener{Listener: server.Listener}
server.Start()
defer server.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}})
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
var welcome protocol.Message
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
t.Fatal(err)
}
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil {
t.Fatal(err)
}
var refused protocol.Message
if err := wsjson.Read(ctx, conn, &refused); err != nil {
t.Fatal(err)
}
if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") {
t.Fatalf("forged peer was accepted: %#v", refused)
}
after, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(before, after) {
t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after)
}
trusted := httptest.NewServer(Handler(ctx, store))
defer trusted.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect")
if err != nil {
t.Fatal(err)
}
defer operator.Close()
if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil {
t.Fatalf("trusted operator transport refused inspection: %v", err)
}
}
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
baseline, err := os.ReadFile("testdata/android-v3-published.md")
if err != nil {
+41 -2
View File
@@ -2,7 +2,9 @@ package server
import (
"context"
"fmt"
"log"
"net"
"net/http"
"strings"
"sync"
@@ -119,7 +121,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
_, _ = w.Write([]byte(`{"healthy":true}`))
})
mux.HandleFunc("GET /connect", func(w http.ResponseWriter, r *http.Request) {
connect := func(w http.ResponseWriter, r *http.Request) {
conn, err := websocket.Accept(w, r, nil)
if err != nil {
return
@@ -268,6 +270,32 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
response.Error = err.Error()
}
}
} else if (request.Type == protocol.InspectResource || request.Type == protocol.ReconcileResource) && len(stores) > 0 {
var err error
response.Type = protocol.ResourceInspectionResult
// Recovery has a separate transport path. A stage/model client may
// also be local, so loopback alone is not an authority boundary.
// The CLI uses /operator-connect; the ordinary /connect path is
// deliberately refused even from loopback.
if r.URL.Path != "/operator-connect" || !isLoopbackPeer(r.RemoteAddr) {
response.Type = protocol.Error
response.Error = "resource recovery requires the local operator transport"
} else if request.Type == protocol.InspectResource {
var inspection protocol.ResourceInspection
inspection, err = stores[0].InspectResource(ctx, request.Resource)
response.ResourceInspection = &inspection
} else {
response.Type = protocol.ResourceReconciled
if request.ResourceRecovery == nil {
err = fmt.Errorf("resource recovery evidence is required")
} else {
err = stores[0].ReconcileResource(ctx, *request.ResourceRecovery)
}
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence) {
var err error
switch request.Type {
@@ -323,10 +351,21 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion)
}
})
}
mux.HandleFunc("GET /connect", connect)
mux.HandleFunc("GET /operator-connect", connect)
return &runHandler{Handler: mux, runner: runs}
}
func isLoopbackPeer(remoteAddr string) bool {
host, _, err := net.SplitHostPort(remoteAddr)
if err != nil {
return false
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
type runHandler struct {
http.Handler
runner *runner