From 27ad7daecd55af469b82bd53f36ce1f83ea3f9df Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 19:18:03 +0100 Subject: [PATCH] test: cover inspected resource recovery boundaries Add executable regressions for a forged forwarded loopback claim over a real WebSocket transport and for replaying an old recovery after FIFO successor handoff.\n\nChecked packaged templates and packaged authoring skill references; neither needs updating. The existing per-item documentation note records shared documentation that the manual docs-fold process must update. --- internal/pipeline/resources_test.go | 242 +++++++++++++++++++++++++ internal/server/resource_cli_test.go | 255 +++++++++++++++++++++++++++ 2 files changed, 497 insertions(+) diff --git a/internal/pipeline/resources_test.go b/internal/pipeline/resources_test.go index 58ae2e6a..c9a1c4b3 100644 --- a/internal/pipeline/resources_test.go +++ b/internal/pipeline/resources_test.go @@ -9,6 +9,8 @@ import ( "sync" "testing" "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" ) func TestResourceDeclaration(t *testing.T) { @@ -286,3 +288,243 @@ func TestResourceRestartQuarantineAndRetention(t *testing.T) { }) } } + +func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + waiter := create("X") + if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil { + t.Fatalf("register waiter: %v", err) + } + // Simulate a runner disappearing while its durable waiter remains. The + // authoritative operation must reject the handoff before audit or cleanup. + delete(s.resourceWaiters, waiter) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"}, + }) + if err == nil { + t.Fatal("reconciled without a live waiter channel") + } + var cleanup, audits, owners int + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil { + t.Fatal(err) + } + if cleanup != 0 || audits != 0 || owners != 1 { + t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners) + } +} + +func TestResourceReconcileTransfersAndAudits(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + waiter := create("X") + _, ch, _, err := s.RequestResource(ctx, waiter) + if err != nil || ch == nil { + t.Fatalf("register waiter: %v", err) + } + if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"}, + }); err != nil { + t.Fatal(err) + } + select { + case next := <-ch: + s.AcceptResource(next) + if next.ResourceVisit != waiter { + t.Fatalf("wrong handoff: %#v", next.ResourceVisit) + } + case <-time.After(time.Second): + t.Fatal("reconciliation did not wake waiter") + } + var audits int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if audits != 1 { + t.Fatalf("recovery audit count=%d", audits) + } +} + +func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + owner := acquireTestResource(t, s, create("X")) + if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + successor := create("X") + _, ch, _, err := s.RequestResource(ctx, successor) + if err != nil || ch == nil { + t.Fatalf("register successor: %v", err) + } + request := protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, + RunID: owner.RunID, + Stage: owner.Stage, + Sequence: owner.Sequence, + Actor: "operator", + Evidence: &protocol.ResourceRecoveryEvidence{ + CleanupConfirmed: true, + NoMutatingAction: true, + Summary: "inspection confirmed no mutating action occurred", + }, + } + if err := s.ReconcileResource(ctx, request); err != nil { + t.Fatal(err) + } + var successorLease ResourceLease + select { + case successorLease = <-ch: + case <-time.After(time.Second): + t.Fatal("reconciliation did not hand off to successor") + } + if successorLease.ResourceVisit != successor { + t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit) + } + if successorLease.token == "" { + t.Fatal("successor capability was empty") + } + s.AcceptResource(successorLease) + + // The old request is an idempotent replay after the handoff. It must not + // match or release the successor's private capability. + if err := s.ReconcileResource(ctx, request); err == nil { + t.Fatal("old recovery replay was accepted against successor owner") + } + inspection, err := s.InspectResource(ctx, owner.Resource) + if err != nil { + t.Fatal(err) + } + if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence { + t.Fatalf("successor was not preserved: %#v", inspection.Owner) + } + if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID { + t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits) + } + if len(s.resourceWaiters) != 0 { + t.Fatal("successor channel registry leaked") + } + var owners, waiters, audits int + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil { + t.Fatal(err) + } + if owners != 1 || waiters != 0 || audits != 1 { + t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits) + } +} + +func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) { + ctx := context.Background() + test := func(t *testing.T, setup func(*Store, ResourceLease) int) { + s, create := resourceStore(t) + owner := acquireTestResource(t, s, create("X")) + expectedCleanup := setup(s, owner) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, + Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"}, + }) + if err == nil { + t.Fatal("reconciled unsafe owner") + } + var owners, audits, cleanup int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if owners != 1 || audits != 0 || cleanup != expectedCleanup { + t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup) + } + } + t.Run("active run", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil { + t.Fatal(err) + } + return 0 + }) + }) + t.Run("active agent", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil { + t.Fatal(err) + } + return 0 + }) + }) + t.Run("already confirmed", func(t *testing.T) { + test(t, func(s *Store, owner ResourceLease) int { + if err := s.SetResourceCleanup(ctx, owner, true); err != nil { + t.Fatal(err) + } + return 1 + }) + }) +} + +func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) { + ctx := context.Background() + for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} { + t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) { + s, create := resourceStore(t) + owner := acquireTestResource(t, s, create("X")) + err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ + Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, + Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary}, + }) + if err == nil { + t.Fatal("accepted malformed evidence") + } + var owners, audits, cleanup int + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { + t.Fatal(err) + } + if owners != 1 || audits != 0 || cleanup != 0 { + t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup) + } + }) + } +} diff --git a/internal/server/resource_cli_test.go b/internal/server/resource_cli_test.go index a377dc70..7147a7f4 100644 --- a/internal/server/resource_cli_test.go +++ b/internal/server/resource_cli_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "net" "net/http/httptest" "os" "path/filepath" @@ -15,11 +16,30 @@ import ( "time" "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/client" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" ) +type fixedRemoteListener struct{ net.Listener } + +func (l fixedRemoteListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return fixedRemoteConn{Conn: conn}, nil +} + +type fixedRemoteConn struct{ net.Conn } + +func (fixedRemoteConn) RemoteAddr() net.Addr { + return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443} +} + // This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE // executor. It never calls inference, a phone, or a deployed server. func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { @@ -110,6 +130,241 @@ func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.") } +func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n" + if _, err := store.Submit(ctx, "operator-resource", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + srv := httptest.NewServer(handler) + defer srv.Close() + normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect") + if err != nil { + t.Fatal(err) + } + if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") { + t.Fatalf("ordinary connect accepted recovery: %v", err) + } + normal.Close() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + runCLI := func(args ...string) string { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil { + t.Fatalf("CLI %v: %v\n%s", args, err, out.String()) + } + return out.String() + } + inspection := runCLI("resource-inspect", visit.Resource, "--json") + if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") { + t.Fatalf("inspection leaked capability: %s", inspection) + } + runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json") + inspection = runCLI("resource-inspect", visit.Resource, "--json") + var recovered protocol.ResourceInspection + if err := json.Unmarshal([]byte(inspection), &recovered); err != nil { + t.Fatal(err) + } + if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" { + t.Fatalf("CLI recovery state was not persisted: %s", inspection) + } +} + +func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + lease, ch, _, err := store.RequestResource(ctx, visit) + if err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + srv := httptest.NewServer(handler) + defer srv.Close() + endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + runCLI := func(args ...string) error { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + err := app.Run(ctx, append([]string{"--url", endpoint}, args...)) + if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") { + t.Fatalf("private capability appeared in CLI output: %s", out.String()) + } + return err + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted wrong owner") + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted stale stage") + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted stale sequence") + } + if err := store.SetResourceCleanup(ctx, lease, true); err != nil { + t.Fatal(err) + } + if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { + t.Fatal("accepted non-quarantined owner") + } + inspection, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 { + t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits)) + } +} + +func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) { + for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} { + if isLoopbackPeer(remote) { + t.Fatalf("accepted non-loopback peer %q", remote) + } + } + for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} { + if !isLoopbackPeer(remote) { + t.Fatalf("rejected loopback peer %q", remote) + } + } + // X-Forwarded-For is deliberately not consulted: a proxy-supplied header + // cannot turn a remote WebSocket peer into the local operator transport. +} + +func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + detail, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} + if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil { + t.Fatalf("acquire resource: channel=%v err=%v", ch, err) + } + if err := store.RecoverRuns(ctx); err != nil { + t.Fatal(err) + } + before, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + + server := httptest.NewUnstartedServer(Handler(ctx, store)) + server.Listener = fixedRemoteListener{Listener: server.Listener} + server.Start() + defer server.Close() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}}) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + var welcome protocol.Message + if err := wsjson.Read(ctx, conn, &welcome); err != nil { + t.Fatal(err) + } + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil { + t.Fatal(err) + } + var refused protocol.Message + if err := wsjson.Read(ctx, conn, &refused); err != nil { + t.Fatal(err) + } + if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") { + t.Fatalf("forged peer was accepted: %#v", refused) + } + after, err := store.InspectResource(ctx, visit.Resource) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, after) { + t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after) + } + + trusted := httptest.NewServer(Handler(ctx, store)) + defer trusted.Close() + normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect") + if err != nil { + t.Fatal(err) + } + if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") { + t.Fatalf("ordinary connect accepted recovery: %v", err) + } + normal.Close() + operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect") + if err != nil { + t.Fatal(err) + } + defer operator.Close() + if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil { + t.Fatalf("trusted operator transport refused inspection: %v", err) + } +} + func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) { baseline, err := os.ReadFile("testdata/android-v3-published.md") if err != nil {