test: cover inspected resource recovery boundaries
Add executable regressions for a forged forwarded loopback claim over a real WebSocket transport and for replaying an old recovery after FIFO successor handoff.\n\nChecked packaged templates and packaged authoring skill references; neither needs updating. The existing per-item documentation note records shared documentation that the manual docs-fold process must update.
This commit is contained in:
@@ -9,6 +9,8 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestResourceDeclaration(t *testing.T) {
|
||||
@@ -286,3 +288,243 @@ func TestResourceRestartQuarantineAndRetention(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
owner := acquireTestResource(t, s, create("X"))
|
||||
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waiter := create("X")
|
||||
if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil {
|
||||
t.Fatalf("register waiter: %v", err)
|
||||
}
|
||||
// Simulate a runner disappearing while its durable waiter remains. The
|
||||
// authoritative operation must reject the handoff before audit or cleanup.
|
||||
delete(s.resourceWaiters, waiter)
|
||||
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
|
||||
Resource: owner.Resource,
|
||||
RunID: owner.RunID,
|
||||
Stage: owner.Stage,
|
||||
Sequence: owner.Sequence,
|
||||
Actor: "operator",
|
||||
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("reconciled without a live waiter channel")
|
||||
}
|
||||
var cleanup, audits, owners int
|
||||
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cleanup != 0 || audits != 0 || owners != 1 {
|
||||
t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceReconcileTransfersAndAudits(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
owner := acquireTestResource(t, s, create("X"))
|
||||
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waiter := create("X")
|
||||
_, ch, _, err := s.RequestResource(ctx, waiter)
|
||||
if err != nil || ch == nil {
|
||||
t.Fatalf("register waiter: %v", err)
|
||||
}
|
||||
if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
|
||||
Resource: owner.Resource,
|
||||
RunID: owner.RunID,
|
||||
Stage: owner.Stage,
|
||||
Sequence: owner.Sequence,
|
||||
Actor: "operator",
|
||||
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case next := <-ch:
|
||||
s.AcceptResource(next)
|
||||
if next.ResourceVisit != waiter {
|
||||
t.Fatalf("wrong handoff: %#v", next.ResourceVisit)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("reconciliation did not wake waiter")
|
||||
}
|
||||
var audits int
|
||||
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if audits != 1 {
|
||||
t.Fatalf("recovery audit count=%d", audits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
owner := acquireTestResource(t, s, create("X"))
|
||||
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
successor := create("X")
|
||||
_, ch, _, err := s.RequestResource(ctx, successor)
|
||||
if err != nil || ch == nil {
|
||||
t.Fatalf("register successor: %v", err)
|
||||
}
|
||||
request := protocol.ResourceRecoveryRequest{
|
||||
Resource: owner.Resource,
|
||||
RunID: owner.RunID,
|
||||
Stage: owner.Stage,
|
||||
Sequence: owner.Sequence,
|
||||
Actor: "operator",
|
||||
Evidence: &protocol.ResourceRecoveryEvidence{
|
||||
CleanupConfirmed: true,
|
||||
NoMutatingAction: true,
|
||||
Summary: "inspection confirmed no mutating action occurred",
|
||||
},
|
||||
}
|
||||
if err := s.ReconcileResource(ctx, request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var successorLease ResourceLease
|
||||
select {
|
||||
case successorLease = <-ch:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("reconciliation did not hand off to successor")
|
||||
}
|
||||
if successorLease.ResourceVisit != successor {
|
||||
t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit)
|
||||
}
|
||||
if successorLease.token == "" {
|
||||
t.Fatal("successor capability was empty")
|
||||
}
|
||||
s.AcceptResource(successorLease)
|
||||
|
||||
// The old request is an idempotent replay after the handoff. It must not
|
||||
// match or release the successor's private capability.
|
||||
if err := s.ReconcileResource(ctx, request); err == nil {
|
||||
t.Fatal("old recovery replay was accepted against successor owner")
|
||||
}
|
||||
inspection, err := s.InspectResource(ctx, owner.Resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence {
|
||||
t.Fatalf("successor was not preserved: %#v", inspection.Owner)
|
||||
}
|
||||
if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID {
|
||||
t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits)
|
||||
}
|
||||
if len(s.resourceWaiters) != 0 {
|
||||
t.Fatal("successor channel registry leaked")
|
||||
}
|
||||
var owners, waiters, audits int
|
||||
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if owners != 1 || waiters != 0 || audits != 1 {
|
||||
t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
test := func(t *testing.T, setup func(*Store, ResourceLease) int) {
|
||||
s, create := resourceStore(t)
|
||||
owner := acquireTestResource(t, s, create("X"))
|
||||
expectedCleanup := setup(s, owner)
|
||||
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
|
||||
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
|
||||
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("reconciled unsafe owner")
|
||||
}
|
||||
var owners, audits, cleanup int
|
||||
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if owners != 1 || audits != 0 || cleanup != expectedCleanup {
|
||||
t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
|
||||
}
|
||||
}
|
||||
t.Run("active run", func(t *testing.T) {
|
||||
test(t, func(s *Store, owner ResourceLease) int {
|
||||
if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return 0
|
||||
})
|
||||
})
|
||||
t.Run("active agent", func(t *testing.T) {
|
||||
test(t, func(s *Store, owner ResourceLease) int {
|
||||
if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return 0
|
||||
})
|
||||
})
|
||||
t.Run("already confirmed", func(t *testing.T) {
|
||||
test(t, func(s *Store, owner ResourceLease) int {
|
||||
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return 1
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} {
|
||||
t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
owner := acquireTestResource(t, s, create("X"))
|
||||
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
|
||||
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
|
||||
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("accepted malformed evidence")
|
||||
}
|
||||
var owners, audits, cleanup int
|
||||
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if owners != 1 || audits != 0 || cleanup != 0 {
|
||||
t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -15,11 +16,30 @@ import (
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/client"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
"github.com/coder/websocket"
|
||||
"github.com/coder/websocket/wsjson"
|
||||
)
|
||||
|
||||
type fixedRemoteListener struct{ net.Listener }
|
||||
|
||||
func (l fixedRemoteListener) Accept() (net.Conn, error) {
|
||||
conn, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fixedRemoteConn{Conn: conn}, nil
|
||||
}
|
||||
|
||||
type fixedRemoteConn struct{ net.Conn }
|
||||
|
||||
func (fixedRemoteConn) RemoteAddr() net.Addr {
|
||||
return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443}
|
||||
}
|
||||
|
||||
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
|
||||
// executor. It never calls inference, a phone, or a deployed server.
|
||||
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
|
||||
@@ -110,6 +130,241 @@ func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
|
||||
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
|
||||
}
|
||||
|
||||
func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"
|
||||
if _, err := store.Submit(ctx, "operator-resource", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
|
||||
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
|
||||
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
|
||||
}
|
||||
if err := store.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
|
||||
t.Fatalf("ordinary connect accepted recovery: %v", err)
|
||||
}
|
||||
normal.Close()
|
||||
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
runCLI := func(args ...string) string {
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
|
||||
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
inspection := runCLI("resource-inspect", visit.Resource, "--json")
|
||||
if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") {
|
||||
t.Fatalf("inspection leaked capability: %s", inspection)
|
||||
}
|
||||
runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json")
|
||||
inspection = runCLI("resource-inspect", visit.Resource, "--json")
|
||||
var recovered protocol.ResourceInspection
|
||||
if err := json.Unmarshal([]byte(inspection), &recovered); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" {
|
||||
t.Fatalf("CLI recovery state was not persisted: %s", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
|
||||
lease, ch, _, err := store.RequestResource(ctx, visit)
|
||||
if err != nil || ch != nil {
|
||||
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
|
||||
}
|
||||
if err := store.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
runCLI := func(args ...string) error {
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
err := app.Run(ctx, append([]string{"--url", endpoint}, args...))
|
||||
if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") {
|
||||
t.Fatalf("private capability appeared in CLI output: %s", out.String())
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
|
||||
t.Fatal("accepted wrong owner")
|
||||
}
|
||||
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
|
||||
t.Fatal("accepted stale stage")
|
||||
}
|
||||
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil {
|
||||
t.Fatal("accepted stale sequence")
|
||||
}
|
||||
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
|
||||
t.Fatal("accepted non-quarantined owner")
|
||||
}
|
||||
inspection, err := store.InspectResource(ctx, visit.Resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 {
|
||||
t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) {
|
||||
for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} {
|
||||
if isLoopbackPeer(remote) {
|
||||
t.Fatalf("accepted non-loopback peer %q", remote)
|
||||
}
|
||||
}
|
||||
for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} {
|
||||
if !isLoopbackPeer(remote) {
|
||||
t.Fatalf("rejected loopback peer %q", remote)
|
||||
}
|
||||
}
|
||||
// X-Forwarded-For is deliberately not consulted: a proxy-supplied header
|
||||
// cannot turn a remote WebSocket peer into the local operator transport.
|
||||
}
|
||||
|
||||
func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
|
||||
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
|
||||
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
|
||||
}
|
||||
if err := store.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := store.InspectResource(ctx, visit.Resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
server := httptest.NewUnstartedServer(Handler(ctx, store))
|
||||
server.Listener = fixedRemoteListener{Listener: server.Listener}
|
||||
server.Start()
|
||||
defer server.Close()
|
||||
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
var welcome protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var refused protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &refused); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") {
|
||||
t.Fatalf("forged peer was accepted: %#v", refused)
|
||||
}
|
||||
after, err := store.InspectResource(ctx, visit.Resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(before, after) {
|
||||
t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after)
|
||||
}
|
||||
|
||||
trusted := httptest.NewServer(Handler(ctx, store))
|
||||
defer trusted.Close()
|
||||
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
|
||||
t.Fatalf("ordinary connect accepted recovery: %v", err)
|
||||
}
|
||||
normal.Close()
|
||||
operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer operator.Close()
|
||||
if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil {
|
||||
t.Fatalf("trusted operator transport refused inspection: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
|
||||
baseline, err := os.ReadFile("testdata/android-v3-published.md")
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user