From 27ad7daecd55af469b82bd53f36ce1f83ea3f9df Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 19:18:03 +0100 Subject: [PATCH 1/2] test: cover inspected resource recovery boundaries Add executable regressions for a forged forwarded loopback claim over a real WebSocket transport and for replaying an old recovery after FIFO successor handoff.\n\nChecked packaged templates and packaged authoring skill references; neither needs updating. The existing per-item documentation note records shared documentation that the manual docs-fold process must update. --- internal/pipeline/resources_test.go | 242 +++++++++++++++++++++++++ internal/server/resource_cli_test.go | 255 +++++++++++++++++++++++++++ 2 files changed, 497 insertions(+) diff --git a/internal/pipeline/resources_test.go b/internal/pipeline/resources_test.go index 58ae2e6a..c9a1c4b3 100644 --- a/internal/pipeline/resources_test.go +++ b/internal/pipeline/resources_test.go @@ -9,6 +9,8 @@ import ( "sync" "testing" "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" ) func TestResourceDeclaration(t *testing.T) { @@ -286,3 +288,243 @@ func TestResourceRestartQuarantineAndRetention(t *testing.T) { }) } } + +func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + waiter := create("X") + if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil { + t.Fatalf("register waiter: %v", err) + } + // Simulate a runner disappearing while its durable waiter remains. The + // authoritative operation must reject the handoff before audit or cleanup. + delete(s.resourceWaiters, waiter) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"}, + }) + if err == nil { + t.Fatal("reconciled without a live waiter channel") + } + var cleanup, audits, owners int + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil { + t.Fatal(err) + } + if cleanup != 0 || audits != 0 || owners != 1 { + t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners) + } +} + +func TestResourceReconcileTransfersAndAudits(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + waiter := create("X") + _, ch, _, err := s.RequestResource(ctx, waiter) + if err != nil || ch == nil { + t.Fatalf("register waiter: %v", err) + } + if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"}, + }); err != nil { + t.Fatal(err) + } + select { + case next := <-ch: + s.AcceptResource(next) + if next.ResourceVisit != waiter { + t.Fatalf("wrong handoff: %#v", next.ResourceVisit) + } + case <-time.After(time.Second): + t.Fatal("reconciliation did not wake waiter") + } + var audits int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if audits != 1 { + t.Fatalf("recovery audit count=%d", audits) + } +} + +func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + successor := create("X") + _, ch, _, err := s.RequestResource(ctx, successor) + if err != nil || ch == nil { + t.Fatalf("register successor: %v", err) + } + request := protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{ + CleanupConfirmed: true, + NoMutatingAction: true, + Summary: "inspection confirmed no mutating action occurred", + }, + } + if err := s.ReconcileResource(ctx, request); err != nil { + t.Fatal(err) + } + var successorLease ResourceLease + select { + case successorLease = <-ch: + case <-time.After(time.Second): + t.Fatal("reconciliation did not hand off to successor") + } + if successorLease.ResourceVisit != successor { + t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit) + } + if successorLease.token == "" { + t.Fatal("successor capability was empty") + } + s.AcceptResource(successorLease) + + // The old request is an idempotent replay after the handoff. It must not + // match or release the successor's private capability. + if err := s.ReconcileResource(ctx, request); err == nil { + t.Fatal("old recovery replay was accepted against successor owner") + } + inspection, err := s.InspectResource(ctx, owner.Resource) + if err != nil { + t.Fatal(err) + } + if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence { + t.Fatalf("successor was not preserved: %#v", inspection.Owner) + } + if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID { + t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("successor channel registry leaked") + } + var owners, waiters, audits int + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if owners != 1 || waiters != 0 || audits != 1 { + t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits) + } +} + +func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) { + ctx := context.Background() + test := func(t *testing.T, setup func(*Store, ResourceLease) int) { + s, create := resourceStore(t) + owner := acquireTestResource(t, s, create("X")) + expectedCleanup := setup(s, owner) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, + Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"}, + }) + if err == nil { + t.Fatal("reconciled unsafe owner") + } + var owners, audits, cleanup int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if owners != 1 || audits != 0 || cleanup != expectedCleanup { + t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup) + } + } + t.Run("active run", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil { + t.Fatal(err) + } + return 0 + }) + }) + t.Run("active agent", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil { + t.Fatal(err) + } + return 0 + }) + }) + t.Run("already confirmed", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if err := s.SetResourceCleanup(ctx, owner, true); err != nil { + t.Fatal(err) + } + return 1 + }) + }) +} + +func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) { + ctx := context.Background() + for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} { + t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) { + s, create := resourceStore(t) + owner := acquireTestResource(t, s, create("X")) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, + Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary}, + }) + if err == nil { + t.Fatal("accepted malformed evidence") + } + var owners, audits, cleanup int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if owners != 1 || audits != 0 || cleanup != 0 { + t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup) + } + }) + } +} diff --git a/internal/server/resource_cli_test.go b/internal/server/resource_cli_test.go index a377dc70..7147a7f4 100644 --- a/internal/server/resource_cli_test.go +++ b/internal/server/resource_cli_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "net" "net/http/httptest" "os" "path/filepath" @@ -15,11 +16,30 @@ import ( "time" "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/client" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" ) +type fixedRemoteListener struct{ net.Listener } + +func (l fixedRemoteListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return fixedRemoteConn{Conn: conn}, nil +} + +type fixedRemoteConn struct{ net.Conn } + +func (fixedRemoteConn) RemoteAddr() net.Addr { + return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443} +} + // This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE // executor. It never calls inference, a phone, or a deployed server. func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { @@ -110,6 +130,241 @@ func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.") } +func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n" + if _, err := store.Submit(ctx, "operator-resource", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + srv := httptest.NewServer(handler) + defer srv.Close() + normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect") + if err != nil { + t.Fatal(err) + } + if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") { + t.Fatalf("ordinary connect accepted recovery: %v", err) + } + normal.Close() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + runCLI := func(args ...string) string { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil { + t.Fatalf("CLI %v: %v\n%s", args, err, out.String()) + } + return out.String() + } + inspection := runCLI("resource-inspect", visit.Resource, "--json") + if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") { + t.Fatalf("inspection leaked capability: %s", inspection) + } + runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json") + inspection = runCLI("resource-inspect", visit.Resource, "--json") + var recovered protocol.ResourceInspection + if err := json.Unmarshal([]byte(inspection), &recovered); err != nil { + t.Fatal(err) + } + if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" { + t.Fatalf("CLI recovery state was not persisted: %s", inspection) + } +} + +func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + lease, ch, _, err := store.RequestResource(ctx, visit) + if err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + srv := httptest.NewServer(handler) + defer srv.Close() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + runCLI := func(args ...string) error { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + err := app.Run(ctx, append([]string{"--url", endpoint}, args...)) + if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") { + t.Fatalf("private capability appeared in CLI output: %s", out.String()) + } + return err + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted wrong owner") + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted stale stage") + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted stale sequence") + } + if err := store.SetResourceCleanup(ctx, lease, true); err != nil { + t.Fatal(err) + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted non-quarantined owner") + } + inspection, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 { + t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits)) + } +} + +func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) { + for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} { + if isLoopbackPeer(remote) { + t.Fatalf("accepted non-loopback peer %q", remote) + } + } + for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} { + if !isLoopbackPeer(remote) { + t.Fatalf("rejected loopback peer %q", remote) + } + } + // X-Forwarded-For is deliberately not consulted: a proxy-supplied header + // cannot turn a remote WebSocket peer into the local operator transport. +} + +func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + before, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + + server := httptest.NewUnstartedServer(Handler(ctx, store)) + server.Listener = fixedRemoteListener{Listener: server.Listener} + server.Start() + defer server.Close() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}}) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + var welcome protocol.Message + if err := wsjson.Read(ctx, conn, &welcome); err != nil { + t.Fatal(err) + } + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil { + t.Fatal(err) + } + var refused protocol.Message + if err := wsjson.Read(ctx, conn, &refused); err != nil { + t.Fatal(err) + } + if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") { + t.Fatalf("forged peer was accepted: %#v", refused) + } + after, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, after) { + t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after) + } + + trusted := httptest.NewServer(Handler(ctx, store)) + defer trusted.Close() + normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect") + if err != nil { + t.Fatal(err) + } + if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") { + t.Fatalf("ordinary connect accepted recovery: %v", err) + } + normal.Close() + operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect") + if err != nil { + t.Fatal(err) + } + defer operator.Close() + if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil { + t.Fatalf("trusted operator transport refused inspection: %v", err) + } +} + func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) { baseline, err := os.ReadFile("testdata/android-v3-published.md") if err != nil { From b79d0401f3c696578916253d2ac507cb17f354ec Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 19:24:32 +0100 Subject: [PATCH 2/2] feat: recover inspected quarantined resources Add local operator inspection and reconciliation for quarantined resource leases, including atomic audit-backed FIFO handoff and recovery protocol support.\n\nPreserve capability tokens in the authoritative store and reject non-loopback or ordinary client transport for recovery mutations.\n\nChecked packaged templates and packaged authoring skill references; none need updating. The task-owned docs note records shared README and skill reference follow-up for docs-fold. --- docs/notes/inspected-resource-recovery.md | 19 ++ internal/cli/app.go | 79 +++++++- internal/cli/commands.go | 68 +++---- internal/pipeline/resources.go | 217 ++++++++++++++++++++-- internal/pipeline/runs_schema.go | 6 + internal/protocol/message.go | 37 ++-- internal/protocol/resources.go | 46 +++++ internal/server/websocket.go | 43 ++++- 8 files changed, 444 insertions(+), 71 deletions(-) create mode 100644 docs/notes/inspected-resource-recovery.md create mode 100644 internal/protocol/resources.go diff --git a/docs/notes/inspected-resource-recovery.md b/docs/notes/inspected-resource-recovery.md new file mode 100644 index 00000000..b3986890 --- /dev/null +++ b/docs/notes/inspected-resource-recovery.md @@ -0,0 +1,19 @@ +# Inspected resource recovery + +The operator recovery contract now keeps quarantined resource ownership and its +audit transactionally coupled to FIFO handoff. Inspection is redacted and +reconciliation is accepted only over a loopback operator connection with the +exact owner visit plus positive cleanup/restoration evidence. The persisted +capability remains private to the authoritative live store; missing waiter +channels fail closed without changing owner or audit state. + +Use the configured `PIPELINES_URL` endpoint, not a disposable URL. First run +the redacted resource inspection, then reconcile only after the parked visit +has been inspected and quiesced, supplying its exact run/stage/sequence and +explicit evidence (including `no mutating action occurred` where applicable). +The old deployed server does not support this action; deploy a compatible +server before attempting recovery. This is an operator trust boundary, not an +OS sandbox or unrestricted authorization grant. + +The shared README and packaged OpenCode skill references should be updated by +the manual `docs-fold` process; they are intentionally not edited here. diff --git a/internal/cli/app.go b/internal/cli/app.go index 5e26e393..e25c493c 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -7,6 +7,7 @@ import ( "flag" "fmt" "io" + "net/url" "os" "os/exec" "path/filepath" @@ -36,7 +37,7 @@ type App struct { type options struct { harness string url, name, prompt, title, template, workspace, paths, key, opencode, model string - option, feedback string + option, feedback, evidence, actor string output string conflict string kind string @@ -45,7 +46,9 @@ type options struct { keepSet bool all bool json, source, noOpen, yes, allow, createOnly, force, wait bool - version, expected, limit int + version, expected, limit, recoverySequence int + recoveryRun int64 + recoveryStage, resource string timeout, waitTimeout time.Duration stageModels stageModelFlags } @@ -177,6 +180,13 @@ func (a App) flags(command string) (*flag.FlagSet, *options) { case "decide": f.StringVar(&o.option, "option", "", "Option key to choose among the waiting stage's edges (required)") f.StringVar(&o.feedback, "feedback", "", "Optional free-text feedback recorded with the decision") + case "resource-inspect": + f.StringVar(&o.resource, "resource", "", "Exclusive resource key") + case "resource-reconcile": + f.StringVar(&o.evidence, "evidence", "", "Bounded positive cleanup/restoration evidence (required)") + f.StringVar(&o.actor, "actor", "operator", "Bounded operator identity") + f.BoolVar(&o.allow, "no-mutating-action", false, "Evidence explicitly records that no mutating action occurred") + f.BoolVar(&o.yes, "yes", false, "Confirm reconciliation without prompting") case "debug": f.StringVar(&o.harness, "harness", "", "Debugging harness: configured opencode/pi, or nvim") f.BoolVar(&o.yes, "yes", false, "Approve opening a debugging session without prompting") @@ -462,8 +472,16 @@ func (a App) Run(ctx context.Context, args []string) error { if requestTimeout <= 0 { requestTimeout = 30 * time.Second } + connectURL := o.url + if command == "resource-inspect" || command == "resource-reconcile" { + var transportErr error + connectURL, transportErr = operatorTransportURL(o.url) + if transportErr != nil { + return transportErr + } + } dialCtx, cancel := context.WithTimeout(ctx, requestTimeout) - c, err := client.Open(dialCtx, o.url) + c, err := client.Open(dialCtx, connectURL) cancel() if err != nil { if command == "status" && len(pos) == 0 { @@ -897,6 +915,51 @@ func (a App) Run(ctx context.Context, args []string) error { } } return fmt.Errorf("template %q not found", key) + case "resource-inspect": + if len(pos) > 1 { + return fmt.Errorf("resource-inspect takes one resource key") + } + if o.resource == "" && len(pos) == 1 { + o.resource = pos[0] + } + if strings.TrimSpace(o.resource) == "" { + return fmt.Errorf("resource-inspect requires a resource key") + } + r, err := call(protocol.Message{Type: protocol.InspectResource, Resource: o.resource}) + if err != nil { + return err + } + if r.ResourceInspection == nil { + return fmt.Errorf("server did not return resource inspection") + } + return a.output(r.ResourceInspection) + case "resource-reconcile": + if len(pos) != 4 { + return fmt.Errorf("resource-reconcile requires resource, run-id, stage and sequence") + } + runID, err := strconv.ParseInt(pos[1], 10, 64) + if err != nil || runID <= 0 { + return fmt.Errorf("run ID must be positive") + } + sequence, err := strconv.Atoi(pos[3]) + if err != nil || sequence <= 0 { + return fmt.Errorf("sequence must be positive") + } + if strings.TrimSpace(o.evidence) == "" { + return fmt.Errorf("--evidence is required") + } + if err := a.confirmAction(fmt.Sprintf("Reconcile quarantined resource %s for run #%d?", pos[0], runID), o.yes); err != nil { + return err + } + r, err := call(protocol.Message{Type: protocol.ReconcileResource, Resource: pos[0], ResourceRecovery: &protocol.ResourceRecoveryRequest{Resource: pos[0], RunID: runID, Stage: pos[2], Sequence: sequence, Actor: o.actor, Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: o.allow, Summary: o.evidence}}}) + if err != nil { + return err + } + if o.json { + return a.output(r) + } + _, err = fmt.Fprintf(a.Out, "Reconciled resource %s for run #%d\n", pos[0], runID) + return err case "inspect", "cancel", "retry": value, err := one() if err != nil { @@ -1039,6 +1102,16 @@ func (a App) Run(ctx context.Context, args []string) error { return nil } +func operatorTransportURL(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return "", fmt.Errorf("invalid configured pipeline URL") + } + u.Path = "/operator-connect" + u.RawPath = "" + return u.String(), nil +} + func (a App) output(value any) error { encoder := json.NewEncoder(a.Out) encoder.SetIndent("", " ") diff --git a/internal/cli/commands.go b/internal/cli/commands.go index a490e56c..a78a8d2d 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -2,39 +2,41 @@ package cli // Help is also the public command inventory for agents and parity checks. var usage = map[string]string{ - "provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]", - "new": "new [name] [prompt] [--template KEY] [--no-open | --json]", - "edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]", - "drafts": "drafts [--json]", - "draft-delete": "draft-delete [--json]", - "validate": "validate [--json]", - "submit": "submit [--name NAME] [--create-only] [--yes] [--json]", - "list": "list [--json]", - "search": "search QUERY [--type KIND] [--limit N] [--json]", - "show": "show [--version N] [--json]", - "versions": "versions [--json]", - "export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]", - "import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]", - "templates": "templates [--json]", - "template": "template [--source | --json]", - "run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]", - "runs": "runs [--pipeline NAME] [--json]", - "inspect": "inspect [--json]", - "wait": "wait [run-id ...] [--timeout DURATION] [--json]", - "cancel": "cancel [--json]", - "decide": "decide [stage] --option KEY [--feedback TEXT] [--json]", - "retry": "retry [--json [--wait [--wait-timeout DURATION]]]", - "title": "title \"\" [--json]", - "debug": "debug <run-id> [--json | --no-open]", - "chat-adopt": "chat-adopt <session-id> [--json]", - "connection": "connection [--json]", - "ping": "ping [--json]", - "version": "version [--json]", - "status": "status [--json]", - "stats": "stats [--json]", - "rpc": "rpc <request.json|->", - "retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]", - "tui": "tui", + "provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]", + "new": "new [name] [prompt] [--template KEY] [--no-open | --json]", + "edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]", + "drafts": "drafts [--json]", + "draft-delete": "draft-delete <draft.md> [--json]", + "validate": "validate <draft.md> [--json]", + "submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]", + "list": "list [--json]", + "search": "search QUERY [--type KIND] [--limit N] [--json]", + "show": "show <name|id> [--version N] [--json]", + "versions": "versions <name|id> [--json]", + "export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]", + "import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]", + "templates": "templates [--json]", + "template": "template <key> [--source | --json]", + "run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]", + "runs": "runs [--pipeline NAME] [--json]", + "inspect": "inspect <run-id> [--json]", + "wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]", + "cancel": "cancel <run-id> [--json]", + "decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]", + "retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]", + "title": "title <run-id> \"<title>\" [--json]", + "debug": "debug <run-id> [--json | --no-open]", + "chat-adopt": "chat-adopt <session-id> [--json]", + "connection": "connection [--json]", + "ping": "ping [--json]", + "version": "version [--json]", + "status": "status [--json]", + "stats": "stats [--json]", + "resource-inspect": "resource-inspect <resource> [--json]", + "resource-reconcile": "resource-reconcile <resource> <run-id> <stage> <sequence> --evidence TEXT [--no-mutating-action] [--yes] [--json]", + "rpc": "rpc <request.json|->", + "retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]", + "tui": "tui", } var notes = map[string]string{ diff --git a/internal/pipeline/resources.go b/internal/pipeline/resources.go index 1a1720b1..5d4907a8 100644 --- a/internal/pipeline/resources.go +++ b/internal/pipeline/resources.go @@ -3,15 +3,184 @@ package pipeline import ( "context" "crypto/rand" + "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "fmt" + "strings" "time" + "unicode/utf8" "git.a-wan.net/alan/pipelines/internal/protocol" ) +const maxRecoveryText = 512 + +func boundedRecoveryText(value string) string { + value = strings.TrimSpace(value) + return value +} + +// InspectResource exposes only bounded state; capability tokens never leave this +// package or enter a protocol value. +func (s *Store) InspectResource(ctx context.Context, resource string) (protocol.ResourceInspection, error) { + if !ValidResourceKey(resource) { + return protocol.ResourceInspection{}, fmt.Errorf("invalid resource") + } + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + out := protocol.ResourceInspection{Resource: resource, Waiters: []protocol.ResourceWaiter{}, Audits: []protocol.ResourceRecoveryAudit{}} + var runID int64 + var stage string + var seq, cleanup int + err := s.db.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, resource).Scan(&runID, &stage, &seq, &cleanup) + if err == nil { + var active int + if err := s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil { + return out, err + } + out.Owner = &protocol.ResourceOwner{RunID: runID, Stage: stage, Sequence: seq, CleanupConfirmed: cleanup != 0, Quarantined: cleanup == 0, AgentActive: active != 0} + } else if err != sql.ErrNoRows { + return out, err + } + rows, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,queued_at FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 32`, resource) + if err != nil { + return out, err + } + for rows.Next() { + var w protocol.ResourceWaiter + if err := rows.Scan(&w.RunID, &w.Stage, &w.Sequence, &w.QueuedAt); err != nil { + rows.Close() + return out, err + } + out.Waiters = append(out.Waiters, w) + } + if err := rows.Err(); err != nil { + rows.Close() + return out, err + } + rows.Close() + audits, err := s.db.QueryContext(ctx, `SELECT run_id,stage_key,sequence,actor,source,evidence,result,created_at FROM resource_recovery_audit WHERE resource_key=? ORDER BY rowid DESC LIMIT 16`, resource) + if err != nil { + return out, err + } + defer audits.Close() + for audits.Next() { + var a protocol.ResourceRecoveryAudit + a.Resource = resource + if err := audits.Scan(&a.RunID, &a.Stage, &a.Sequence, &a.Actor, &a.Source, &a.Evidence, &a.Result, &a.CreatedAt); err != nil { + audits.Close() + return out, err + } + out.Audits = append(out.Audits, a) + } + return out, audits.Err() +} + +// ReconcileResource is the sole operator recovery mutation. The persisted +// capability is compared only inside this authoritative store operation. +func (s *Store) ReconcileResource(ctx context.Context, request protocol.ResourceRecoveryRequest) error { + if !ValidResourceKey(request.Resource) || request.RunID <= 0 || request.Sequence <= 0 || !identifier.MatchString(request.Stage) { + return fmt.Errorf("invalid resource recovery identity") + } + if request.Evidence == nil || !request.Evidence.CleanupConfirmed || !validRecoveryEvidence(request.Evidence.Summary) { + return fmt.Errorf("positive cleanup/restoration evidence is required") + } + if strings.TrimSpace(request.Actor) == "" || len(request.Actor) > maxRecoveryText { + return fmt.Errorf("bounded actor is required") + } + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var token, stage string + var runID int64 + var seq, cleanup int + err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token,cleanup_confirmed FROM resource_owners WHERE resource_key=?`, request.Resource).Scan(&runID, &stage, &seq, &token, &cleanup) + if err == sql.ErrNoRows { + var count int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND result='reconciled'`, request.Resource, request.RunID, request.Stage, request.Sequence).Scan(&count); err != nil { + return err + } + if count > 0 { + return nil + } + return fmt.Errorf("resource owner not found") + } + if err != nil { + return err + } + if runID != request.RunID || stage != request.Stage || seq != request.Sequence { + return fmt.Errorf("resource owner visit is stale") + } + if cleanup != 0 { + return fmt.Errorf("resource owner is not quarantined") + } + var status string + if err := tx.QueryRowContext(ctx, `SELECT status FROM pipeline_runs WHERE id=?`, runID).Scan(&status); err != nil { + return err + } + if status == protocol.RunStatusQueued || status == protocol.RunStatusWaiting || status == protocol.RunStatusRunning { + return fmt.Errorf("owning run is still active") + } + var active int + if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(agent_active),0) FROM run_steps WHERE run_id=? AND sequence=?`, runID, seq).Scan(&active); err != nil { + return err + } + if active != 0 { + return fmt.Errorf("owning agent is still active") + } + evidence := boundedRecoveryText(request.Evidence.Summary) + if request.Evidence.NoMutatingAction { + evidence = boundedRecoveryText("no mutating action occurred; " + evidence) + } + evidenceHash := sha256.Sum256([]byte(evidence)) + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_recovery_audit(resource_key,run_id,stage_key,sequence,actor,source,evidence,result) VALUES(?,?,?,?,?,?,?,?)`, request.Resource, runID, stage, seq, boundedRecoveryText(request.Actor), "loopback-operator", "sha256:"+hex.EncodeToString(evidenceHash[:]), "reconciled"); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=1,diagnostic='' WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=0`, request.Resource, runID, stage, seq, token); err != nil { + return err + } + owner := ResourceLease{ResourceVisit: ResourceVisit{Resource: request.Resource, RunID: runID, Stage: stage, Sequence: seq}, token: token} + if _, err := deleteResourceOwner(ctx, tx, owner); err != nil { + return err + } + next, err := s.handoffResourceTx(ctx, tx, request.Resource) + if err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + if next.token != "" { + s.resourceWaiters[next.ResourceVisit] <- next + } + return nil +} + +func validRecoveryEvidence(value string) bool { + value = strings.TrimSpace(value) + if value == "" || len(value) > maxRecoveryText || !utf8.ValidString(value) { + return false + } + if strings.EqualFold(value, "true") || strings.EqualFold(value, "false") || strings.EqualFold(value, "null") { + return false + } + if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") { + return false + } + for _, r := range value { + if r == '\x00' || (r < 0x20 && r != '\t' && r != '\n' && r != '\r') { + return false + } + } + return true +} + // ResourceVisit identifies a pending visit, before it consumes execution accounting. type ResourceVisit struct { Resource string @@ -208,26 +377,10 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) if n == 0 { return nil } - var next ResourceLease - next.Resource = lease.Resource - err = tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, lease.Resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token) - if err != nil && err != sql.ErrNoRows { + next, err := s.handoffResourceTx(ctx, tx, lease.Resource) + if err != nil { return err } - if err == nil { - if s.resourceWaiters[next.ResourceVisit] == nil { - return fmt.Errorf("resource waiter has no live runner; refusing handoff") - } - if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, "DELETE FROM resource_waiters WHERE run_id=?", next.RunID); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { - return err - } - } if err := tx.Commit(); err != nil { return err } @@ -237,6 +390,34 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) return nil } +// handoffResourceTx is shared by ordinary release and inspected recovery. It +// validates the in-memory waiter before changing durable ownership so a stale +// server cannot grant a capability to a runner that no longer exists. +func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) { + var next ResourceLease + next.Resource = resource + err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token) + if err == sql.ErrNoRows { + return ResourceLease{}, nil + } + if err != nil { + return ResourceLease{}, err + } + if s.resourceWaiters[next.ResourceVisit] == nil { + return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff") + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil { + return ResourceLease{}, err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil { + return ResourceLease{}, err + } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { + return ResourceLease{}, err + } + return next, nil +} + func deleteResourceOwner(ctx context.Context, tx *sql.Tx, lease ResourceLease) (int64, error) { result, err := tx.ExecContext(ctx, `DELETE FROM resource_owners WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=? AND cleanup_confirmed=1`, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token) if err != nil { diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index 8f861061..2c9414ef 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -73,6 +73,12 @@ CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,q if err := s.addColumnIfMissing("pipeline_runs", "resource_wait", "TEXT NOT NULL DEFAULT ''"); err != nil { return err } + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_recovery_audit ( +resource_key TEXT NOT NULL, run_id INTEGER NOT NULL, stage_key TEXT NOT NULL, +sequence INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL, +evidence TEXT NOT NULL, result TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)`); err != nil { + return err + } return s.migrateRunModels() } diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 1fbaa296..b3b8e12f 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -3,19 +3,23 @@ package protocol import "git.a-wan.net/alan/pipelines/internal/buildinfo" const ( - Ping = "ping" - Pong = "pong" - Error = "error" - Submit = "submit" - Submitted = "submitted" - List = "list_pipelines" - PipelineList = "pipelines" - GetPipeline = "get_pipeline" - Pipeline = "pipeline" - ListTemplates = "list_templates" - Templates = "templates" - Validate = "validate" - Validated = "validated" + Ping = "ping" + Pong = "pong" + Error = "error" + Submit = "submit" + Submitted = "submitted" + List = "list_pipelines" + PipelineList = "pipelines" + GetPipeline = "get_pipeline" + Pipeline = "pipeline" + ListTemplates = "list_templates" + Templates = "templates" + Validate = "validate" + Validated = "validated" + InspectResource = "inspect_resource" + ResourceInspectionResult = "resource_inspection" + ReconcileResource = "reconcile_resource" + ResourceReconciled = "resource_reconciled" ) type Stage struct { @@ -191,6 +195,9 @@ type Message struct { // RunEvent is the server-pushed run state delta (see runevents.go) and // RunIDs the optional subscribe scope, so a client asks for the runs it // displays. Both are optional so older peers ignore them. - RunEvent *RunEventFrame `json:"run_event,omitempty"` - RunIDs []int64 `json:"run_ids,omitempty"` + RunEvent *RunEventFrame `json:"run_event,omitempty"` + RunIDs []int64 `json:"run_ids,omitempty"` + Resource string `json:"resource,omitempty"` + ResourceInspection *ResourceInspection `json:"resource_inspection,omitempty"` + ResourceRecovery *ResourceRecoveryRequest `json:"resource_recovery,omitempty"` } diff --git a/internal/protocol/resources.go b/internal/protocol/resources.go new file mode 100644 index 00000000..4f9c3c53 --- /dev/null +++ b/internal/protocol/resources.go @@ -0,0 +1,46 @@ +package protocol + +type ResourceInspection struct { + Resource string `json:"resource"` + Owner *ResourceOwner `json:"owner,omitempty"` + Waiters []ResourceWaiter `json:"waiters,omitempty"` + Audits []ResourceRecoveryAudit `json:"audits,omitempty"` +} +type ResourceOwner struct { + RunID int64 `json:"run_id"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + CleanupConfirmed bool `json:"cleanup_confirmed"` + Quarantined bool `json:"quarantined"` + AgentActive bool `json:"agent_active"` +} +type ResourceWaiter struct { + RunID int64 `json:"run_id"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + QueuedAt string `json:"queued_at"` +} +type ResourceRecoveryAudit struct { + Resource string `json:"resource"` + RunID int64 `json:"run_id"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + Actor string `json:"actor"` + Source string `json:"source"` + Evidence string `json:"evidence"` + Result string `json:"result"` + CreatedAt string `json:"created_at"` +} +type ResourceRecoveryEvidence struct { + CleanupConfirmed bool `json:"cleanup_confirmed"` + NoMutatingAction bool `json:"no_mutating_action"` + Summary string `json:"summary"` +} +type ResourceRecoveryRequest struct { + Resource string `json:"resource"` + RunID int64 `json:"run_id"` + Stage string `json:"stage"` + Sequence int `json:"sequence"` + Evidence *ResourceRecoveryEvidence `json:"evidence,omitempty"` + Actor string `json:"actor"` +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 5e598223..152d8434 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -2,7 +2,9 @@ package server import ( "context" + "fmt" "log" + "net" "net/http" "strings" "sync" @@ -119,7 +121,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } _, _ = w.Write([]byte(`{"healthy":true}`)) }) - mux.HandleFunc("GET /connect", func(w http.ResponseWriter, r *http.Request) { + connect := func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) if err != nil { return @@ -268,6 +270,32 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin response.Error = err.Error() } } + } else if (request.Type == protocol.InspectResource || request.Type == protocol.ReconcileResource) && len(stores) > 0 { + var err error + response.Type = protocol.ResourceInspectionResult + // Recovery has a separate transport path. A stage/model client may + // also be local, so loopback alone is not an authority boundary. + // The CLI uses /operator-connect; the ordinary /connect path is + // deliberately refused even from loopback. + if r.URL.Path != "/operator-connect" || !isLoopbackPeer(r.RemoteAddr) { + response.Type = protocol.Error + response.Error = "resource recovery requires the local operator transport" + } else if request.Type == protocol.InspectResource { + var inspection protocol.ResourceInspection + inspection, err = stores[0].InspectResource(ctx, request.Resource) + response.ResourceInspection = &inspection + } else { + response.Type = protocol.ResourceReconciled + if request.ResourceRecovery == nil { + err = fmt.Errorf("resource recovery evidence is required") + } else { + err = stores[0].ReconcileResource(ctx, *request.ResourceRecovery) + } + } + if err != nil { + response.Type = protocol.Error + response.Error = err.Error() + } } else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence) { var err error switch request.Type { @@ -323,10 +351,21 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } log.Printf("response sent: client=%s type=%q id=%q time=%q error=%q version=%d", r.RemoteAddr, response.Type, response.ID, response.Time, response.Error, selectedVersion) } - }) + } + mux.HandleFunc("GET /connect", connect) + mux.HandleFunc("GET /operator-connect", connect) return &runHandler{Handler: mux, runner: runs} } +func isLoopbackPeer(remoteAddr string) bool { + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return false + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + type runHandler struct { http.Handler runner *runner