test: cover inspected resource recovery boundaries

Add executable regressions for a forged forwarded loopback claim over a real WebSocket transport and for replaying an old recovery after FIFO successor handoff.\n\nChecked packaged templates and packaged authoring skill references; neither needs updating. The existing per-item documentation note records shared documentation that the manual docs-fold process must update.
This commit is contained in:
2026-10-18 19:18:03 +01:00
parent 5946f1abc8
commit 27ad7daecd
2 changed files with 497 additions and 0 deletions
+242
View File
@@ -9,6 +9,8 @@ import (
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestResourceDeclaration(t *testing.T) {
@@ -286,3 +288,243 @@ func TestResourceRestartQuarantineAndRetention(t *testing.T) {
})
}
}
func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
waiter := create("X")
if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil {
t.Fatalf("register waiter: %v", err)
}
// Simulate a runner disappearing while its durable waiter remains. The
// authoritative operation must reject the handoff before audit or cleanup.
delete(s.resourceWaiters, waiter)
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
})
if err == nil {
t.Fatal("reconciled without a live waiter channel")
}
var cleanup, audits, owners int
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
t.Fatal(err)
}
if cleanup != 0 || audits != 0 || owners != 1 {
t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners)
}
}
func TestResourceReconcileTransfersAndAudits(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
waiter := create("X")
_, ch, _, err := s.RequestResource(ctx, waiter)
if err != nil || ch == nil {
t.Fatalf("register waiter: %v", err)
}
if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
}); err != nil {
t.Fatal(err)
}
select {
case next := <-ch:
s.AcceptResource(next)
if next.ResourceVisit != waiter {
t.Fatalf("wrong handoff: %#v", next.ResourceVisit)
}
case <-time.After(time.Second):
t.Fatal("reconciliation did not wake waiter")
}
var audits int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if audits != 1 {
t.Fatalf("recovery audit count=%d", audits)
}
}
func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
owner := acquireTestResource(t, s, create("X"))
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
successor := create("X")
_, ch, _, err := s.RequestResource(ctx, successor)
if err != nil || ch == nil {
t.Fatalf("register successor: %v", err)
}
request := protocol.ResourceRecoveryRequest{
Resource: owner.Resource,
RunID: owner.RunID,
Stage: owner.Stage,
Sequence: owner.Sequence,
Actor: "operator",
Evidence: &protocol.ResourceRecoveryEvidence{
CleanupConfirmed: true,
NoMutatingAction: true,
Summary: "inspection confirmed no mutating action occurred",
},
}
if err := s.ReconcileResource(ctx, request); err != nil {
t.Fatal(err)
}
var successorLease ResourceLease
select {
case successorLease = <-ch:
case <-time.After(time.Second):
t.Fatal("reconciliation did not hand off to successor")
}
if successorLease.ResourceVisit != successor {
t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit)
}
if successorLease.token == "" {
t.Fatal("successor capability was empty")
}
s.AcceptResource(successorLease)
// The old request is an idempotent replay after the handoff. It must not
// match or release the successor's private capability.
if err := s.ReconcileResource(ctx, request); err == nil {
t.Fatal("old recovery replay was accepted against successor owner")
}
inspection, err := s.InspectResource(ctx, owner.Resource)
if err != nil {
t.Fatal(err)
}
if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence {
t.Fatalf("successor was not preserved: %#v", inspection.Owner)
}
if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID {
t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits)
}
if len(s.resourceWaiters) != 0 {
t.Fatal("successor channel registry leaked")
}
var owners, waiters, audits int
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
t.Fatal(err)
}
if owners != 1 || waiters != 0 || audits != 1 {
t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits)
}
}
func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) {
ctx := context.Background()
test := func(t *testing.T, setup func(*Store, ResourceLease) int) {
s, create := resourceStore(t)
owner := acquireTestResource(t, s, create("X"))
expectedCleanup := setup(s, owner)
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"},
})
if err == nil {
t.Fatal("reconciled unsafe owner")
}
var owners, audits, cleanup int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if owners != 1 || audits != 0 || cleanup != expectedCleanup {
t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
}
}
t.Run("active run", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil {
t.Fatal(err)
}
return 0
})
})
t.Run("active agent", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil {
t.Fatal(err)
}
return 0
})
})
t.Run("already confirmed", func(t *testing.T) {
test(t, func(s *Store, owner ResourceLease) int {
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
t.Fatal(err)
}
return 1
})
})
}
func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) {
ctx := context.Background()
for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} {
t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) {
s, create := resourceStore(t)
owner := acquireTestResource(t, s, create("X"))
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary},
})
if err == nil {
t.Fatal("accepted malformed evidence")
}
var owners, audits, cleanup int
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
t.Fatal(err)
}
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
t.Fatal(err)
}
if owners != 1 || audits != 0 || cleanup != 0 {
t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
}
})
}
}
+255
View File
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"encoding/json"
"fmt"
"net"
"net/http/httptest"
"os"
"path/filepath"
@@ -15,11 +16,30 @@ import (
"time"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/client"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
type fixedRemoteListener struct{ net.Listener }
func (l fixedRemoteListener) Accept() (net.Conn, error) {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return fixedRemoteConn{Conn: conn}, nil
}
type fixedRemoteConn struct{ net.Conn }
func (fixedRemoteConn) RemoteAddr() net.Addr {
return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443}
}
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
// executor. It never calls inference, a phone, or a deployed server.
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
@@ -110,6 +130,241 @@ func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
}
func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"
if _, err := store.Submit(ctx, "operator-resource", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) string {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
}
return out.String()
}
inspection := runCLI("resource-inspect", visit.Resource, "--json")
if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") {
t.Fatalf("inspection leaked capability: %s", inspection)
}
runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json")
inspection = runCLI("resource-inspect", visit.Resource, "--json")
var recovered protocol.ResourceInspection
if err := json.Unmarshal([]byte(inspection), &recovered); err != nil {
t.Fatal(err)
}
if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" {
t.Fatalf("CLI recovery state was not persisted: %s", inspection)
}
}
func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
lease, ch, _, err := store.RequestResource(ctx, visit)
if err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) error {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
err := app.Run(ctx, append([]string{"--url", endpoint}, args...))
if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") {
t.Fatalf("private capability appeared in CLI output: %s", out.String())
}
return err
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted wrong owner")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale stage")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale sequence")
}
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
t.Fatal(err)
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted non-quarantined owner")
}
inspection, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 {
t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits))
}
}
func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) {
for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} {
if isLoopbackPeer(remote) {
t.Fatalf("accepted non-loopback peer %q", remote)
}
}
for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} {
if !isLoopbackPeer(remote) {
t.Fatalf("rejected loopback peer %q", remote)
}
}
// X-Forwarded-For is deliberately not consulted: a proxy-supplied header
// cannot turn a remote WebSocket peer into the local operator transport.
}
func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
before, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
server := httptest.NewUnstartedServer(Handler(ctx, store))
server.Listener = fixedRemoteListener{Listener: server.Listener}
server.Start()
defer server.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}})
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
var welcome protocol.Message
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
t.Fatal(err)
}
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil {
t.Fatal(err)
}
var refused protocol.Message
if err := wsjson.Read(ctx, conn, &refused); err != nil {
t.Fatal(err)
}
if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") {
t.Fatalf("forged peer was accepted: %#v", refused)
}
after, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(before, after) {
t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after)
}
trusted := httptest.NewServer(Handler(ctx, store))
defer trusted.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect")
if err != nil {
t.Fatal(err)
}
defer operator.Close()
if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil {
t.Fatalf("trusted operator transport refused inspection: %v", err)
}
}
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
baseline, err := os.ReadFile("testdata/android-v3-published.md")
if err != nil {