package pipeline import ( "context" "encoding/json" "fmt" "path/filepath" "strings" "sync" "testing" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) func TestResourceDeclaration(t *testing.T) { base := Template([]string{"work"}, true) for _, value := range []string{`{"exclusive":"phone-X"}`, `{ "exclusive" : "A_1" }`} { graph, err := Parse(base + "```resources\n" + value + "\n```\n") if err != nil || graph.Stages[0].Resources == nil { t.Fatalf("%s: %v", value, err) } if _, ok := StageDefinition(graph.Stages[0]).Blocks["resources"]; ok { t.Fatal("runner control reached executor") } encoded, _ := json.Marshal(graph) if !strings.Contains(string(encoded), `"resources"`) { t.Fatal("resource lost in persisted graph") } } for _, value := range []string{`{}`, `null`, `[]`, `{"exclusive":""}`, `{"exclusive":"a b"}`, `{"exclusive":"a","exclusive":"b"}`, `{"exclusive":"a","extra":1}`, `{"exclusive":["a","b"]}`, fmt.Sprintf(`{"exclusive":%q}`, strings.Repeat("a", 129))} { if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil { t.Fatalf("accepted %s", value) } } human := "# stage gate\n```prompt\nChoose\n```\n```configs\ntype=human\n```\n```edges\ngo -> #success\n```\n```resources\n{\"exclusive\":\"phone\"}\n```\n" if _, err := Parse(human); err == nil { t.Fatal("human acquired a resource") } } func TestResourcePoolDeclaration(t *testing.T) { base := Template([]string{"work"}, true) graph, err := Parse(base + "```resources\n{\"pool\":\"android-targets\"}\n```\n") if err != nil || graph.Stages[0].Resources == nil || graph.Stages[0].Resources.Pool != "android-targets" { t.Fatalf("pool declaration: %#v, %v", graph.Stages[0].Resources, err) } for _, value := range []string{`{"exclusive":"phone","pool":"targets"}`, `{"pool":""}`, `{"pool":"a b"}`} { if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil { t.Fatalf("accepted invalid pool declaration %s", value) } } } func TestResourcePoolAllocatesConcreteMembersAndExcludesDirectOwner(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "android-targets", Members: []protocol.ResourcePoolMember{ {Member: "phone", Enabled: true, Metadata: map[string]string{"kind": "phone"}}, {Member: "emulator", Enabled: true, Metadata: map[string]string{"kind": "emulator"}}, }}); err != nil { t.Fatal(err) } phone := acquireTestResource(t, s, create("phone")) visit := create("android-targets") lease, ch, _, err := s.RequestResourcePool(ctx, "android-targets", visit) if err != nil || ch != nil || lease.Target == nil || lease.Target.Member != "emulator" { t.Fatalf("pool selected %#v, channel=%v, err=%v", lease.Target, ch, err) } if lease.Target.Metadata["kind"] != "emulator" { t.Fatal("missing bounded target metadata") } releaseTestResource(t, s, lease) releaseTestResource(t, s, phone) } func TestResourcePoolFIFOWakeAndQuarantineIsolation(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ {Member: "held", Enabled: true}, {Member: "free", Enabled: true}, {Member: "bad", Enabled: false, Quarantined: true}, }}); err != nil { t.Fatal(err) } held := acquireTestResource(t, s, create("held")) first := create("targets") firstLease, firstCh, _, err := s.RequestResourcePool(ctx, "targets", first) if err != nil || firstCh != nil || firstLease.Target == nil { t.Fatalf("expected immediate free member: %v", err) } // The second request waits once both concrete members are held. second := create("targets") _, secondCh, _, err := s.RequestResourcePool(ctx, "targets", second) if err != nil || secondCh == nil { t.Fatalf("expected pool waiter: %v", err) } releaseTestResource(t, s, firstLease) select { case next := <-secondCh: if next.Target == nil || next.Target.Member != "free" { t.Fatalf("wrong handoff: %#v", next.Target) } s.AcceptResource(next) releaseTestResource(t, s, next) case <-time.After(time.Second): t.Fatal("pool waiter not woken") } releaseTestResource(t, s, held) } func TestResourcePoolHandoffUsesAnotherMemberWhenReleasedTargetQuarantined(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() pool := protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ {Member: "phone", Enabled: true}, {Member: "emulator", Enabled: false}, }} if err := s.ConfigureResourcePool(ctx, pool); err != nil { t.Fatal(err) } owner, ready, _, err := s.RequestResourcePool(ctx, pool.Name, create(pool.Name)) if err != nil || ready != nil || owner.Target.Member != "phone" { t.Fatalf("owner: %+v %v", owner, err) } _, waiter, _, err := s.RequestResourcePool(ctx, pool.Name, create(pool.Name)) if err != nil || waiter == nil { t.Fatalf("waiter: %v", err) } pool.Members[0].Quarantined = true pool.Members[1].Enabled = true if err := s.ConfigureResourcePool(ctx, pool); err != nil { t.Fatal(err) } releaseTestResource(t, s, owner) select { case next := <-waiter: if next.Target == nil || next.Target.Member != "emulator" { t.Fatalf("wrong target: %+v", next.Target) } s.AcceptResource(next) releaseTestResource(t, s, next) case <-time.After(time.Second): t.Fatal("quarantined released phone blocked the free emulator") } } func TestResourcePoolRejectsStaleVisitForExistingOwner(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ {Member: "phone", Enabled: true}, }}); err != nil { t.Fatal(err) } owner := acquireTestResource(t, s, create("phone")) stale := owner.ResourceVisit stale.Resource = "targets" stale.Sequence++ if _, ch, _, err := s.RequestResourcePool(ctx, "targets", stale); err == nil || ch != nil { t.Fatalf("stale pool visit was accepted: channel=%v err=%v", ch, err) } if err := s.SetResourceCleanup(ctx, owner, true); err != nil { t.Fatal(err) } if err := s.ReleaseResource(ctx, owner); err != nil { t.Fatal(err) } } func TestResourcePoolAndDirectVisitsCannotBeReinterpreted(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true}}}); err != nil { t.Fatal(err) } pooled := create("targets") lease, ch, _, err := s.RequestResourcePool(ctx, "targets", pooled) if err != nil || ch != nil { t.Fatalf("pool acquire: %v %v", ch, err) } if _, ch, _, err := s.RequestResource(ctx, ResourceVisit{Resource: "phone", RunID: pooled.RunID, Stage: pooled.Stage, Sequence: pooled.Sequence}); err == nil || ch != nil { t.Fatalf("pooled owner was accepted as direct lease: channel=%v err=%v", ch, err) } releaseTestResource(t, s, lease) } func TestResourcePoolConfigurationAndOwnerSurviveReopen(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ {Member: "phone", Enabled: true, Metadata: map[string]string{"kind": "phone"}}, }}); err != nil { t.Fatal(err) } visit := create("targets") owner, ch, _, err := s.RequestResourcePool(ctx, "targets", visit) if err != nil || ch != nil { t.Fatalf("pool acquire: %v %v", ch, err) } path := s.path if err := s.Close(); err != nil { t.Fatal(err) } reopened, err := Open(path) if err != nil { t.Fatal(err) } defer reopened.Close() recovered, ch, _, err := reopened.RequestResourcePool(ctx, "targets", visit) if err != nil || ch != nil || recovered.Target == nil || recovered.Target.Member != "phone" || recovered.Target.Metadata["kind"] != "phone" { t.Fatalf("reopened pool owner not recovered: target=%#v channel=%v err=%v", recovered.Target, ch, err) } if err := reopened.SetResourceCleanup(ctx, recovered, true); err != nil { t.Fatal(err) } if err := reopened.ReleaseResource(ctx, recovered); err != nil { t.Fatal(err) } _ = owner } func TestResourcePoolAllocatesDistinctMembersConcurrently(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ {Member: "phone", Enabled: true}, {Member: "emulator", Enabled: true}, }}); err != nil { t.Fatal(err) } visits := []ResourceVisit{create("targets"), create("targets")} leases := make(chan ResourceLease, len(visits)) errs := make(chan error, len(visits)) var wg sync.WaitGroup for _, visit := range visits { wg.Add(1) go func(v ResourceVisit) { defer wg.Done() lease, ch, _, err := s.RequestResourcePool(ctx, "targets", v) if err != nil { errs <- err return } if ch != nil { select { case lease = <-ch: case <-time.After(time.Second): errs <- fmt.Errorf("pool request remained queued") return } s.AcceptResource(lease) } leases <- lease }(visit) } wg.Wait() close(errs) for err := range errs { t.Fatal(err) } got := map[string]bool{} for range visits { lease := <-leases if lease.Target == nil || got[lease.Target.Member] { t.Fatalf("members were not distinct: %#v", lease.Target) } got[lease.Target.Member] = true releaseTestResource(t, s, lease) } } func TestResourcePoolRejectsInvalidDeclarations(t *testing.T) { s, _ := resourceStore(t) ctx := context.Background() cases := []protocol.ResourcePool{ {Name: "missing-members"}, {Name: "duplicate", Members: []protocol.ResourcePoolMember{{Member: "phone"}, {Member: "phone"}}}, {Name: "bad-member", Members: []protocol.ResourcePoolMember{{Member: "not valid"}}}, {Name: "bad-metadata", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{"bad key": "x"}}}}, {Name: "oversized-metadata-key", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{strings.Repeat("k", maxTargetMetadataKey+1): "x"}}}}, } for _, pool := range cases { if err := s.ConfigureResourcePool(ctx, pool); err == nil { t.Fatalf("accepted invalid pool %#v", pool) } } visit := ResourceVisit{Resource: "unknown", RunID: 1, Stage: "work", Sequence: 1} if _, _, _, err := s.RequestResourcePool(ctx, "unknown", visit); err == nil { t.Fatal("accepted unknown pool") } } func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) { t.Helper() ctx := context.Background() s, err := Open(filepath.Join(t.TempDir(), "resources.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { s.Close() }) if _, err := s.Submit(ctx, "resource", Template([]string{"work"}, true)); err != nil { t.Fatal(err) } items, _ := s.List(ctx) d, _ := s.Get(ctx, items[0].ID, 1) count := 0 return s, func(resource string) ResourceVisit { count++ id, _, err := s.CreateRun(ctx, d, fmt.Sprint(count), "workspace", false, nil, nil) if err != nil { t.Fatal(err) } return ResourceVisit{Resource: resource, RunID: id, Stage: "work", Sequence: 1} } } func acquireTestResource(t *testing.T, s *Store, visit ResourceVisit) ResourceLease { t.Helper() l, ch, _, err := s.RequestResource(context.Background(), visit) if err != nil || ch != nil { t.Fatalf("acquire: %v %v", ch, err) } return l } func releaseTestResource(t *testing.T, s *Store, lease ResourceLease) { t.Helper() if err := s.SetResourceCleanup(context.Background(), lease, true); err != nil { t.Fatal(err) } if err := s.ReleaseResource(context.Background(), lease); err != nil { t.Fatal(err) } } func TestResourceFIFOOwnershipCancellationAndRegistry(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() a := acquireTestResource(t, s, create("X")) duplicate := acquireTestResource(t, s, a.ResourceVisit) if duplicate != a { t.Fatal("duplicate acquisition changed capability") } var visits []ResourceVisit var channels []<-chan ResourceLease for i := 0; i < 4; i++ { v := create("X") _, ch, wait, err := s.RequestResource(ctx, v) if err != nil || ch == nil || wait == nil { t.Fatal("missing waiter", err) } _, again, _, err := s.RequestResource(ctx, v) if err != nil || again != ch { t.Fatal("duplicate waiter registered", err) } visits = append(visits, v) channels = append(channels, ch) } if err := s.ReleaseResource(ctx, a); err != nil { t.Fatal(err) } // no cleanup receipt for _, ch := range channels { select { case <-ch: t.Fatal("unconfirmed release granted") default: } } if err := s.SetResourceCleanup(ctx, a, true); err != nil { t.Fatal(err) } for _, corrupt := range []func(*ResourceLease){func(l *ResourceLease) { l.RunID++ }, func(l *ResourceLease) { l.Stage = "other" }, func(l *ResourceLease) { l.Sequence++ }, func(l *ResourceLease) { l.token = "wrong" }} { bad := a corrupt(&bad) if err := s.ReleaseResource(ctx, bad); err != nil { t.Fatal(err) } } for _, ch := range channels { select { case <-ch: t.Fatal("unconfirmed/mismatched release granted") default: } } if err := s.CancelResource(ctx, visits[1]); err != nil { t.Fatal(err) } releaseTestResource(t, s, a) for _, i := range []int{0, 2, 3} { var grant ResourceLease select { case grant = <-channels[i]: case <-time.After(time.Second): t.Fatal("FIFO grant not delivered") } s.AcceptResource(grant) if grant.ResourceVisit != visits[i] { t.Fatal("wrong FIFO owner") } // An old owner's repeated release cannot release the next owner. if err := s.ReleaseResource(ctx, a); err != nil { t.Fatal(err) } for _, j := range []int{2, 3} { if j > i { select { case <-channels[j]: t.Fatal("multiple grants") default: } } } releaseTestResource(t, s, grant) } if len(s.resourceWaiters) != 0 { t.Fatal("channel registry leaked") } for i := 0; i < 30; i++ { owner := acquireTestResource(t, s, create("X")) v := create("X") _, _, _, err := s.RequestResource(ctx, v) if err != nil { t.Fatal(err) } releaseTestResource(t, s, owner) // race window: granted but not accepted if err := s.CancelResource(ctx, v); err != nil { t.Fatal(err) } if len(s.resourceWaiters) != 0 { t.Fatal("grant/cancel leaked channel") } } for i := 0; i < 30; i++ { owner := acquireTestResource(t, s, create("X")) v := create("X") if _, _, _, err := s.RequestResource(ctx, v); err != nil { t.Fatal(err) } if err := s.SetResourceCleanup(ctx, owner, true); err != nil { t.Fatal(err) } var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() if err := s.ReleaseResource(ctx, owner); err != nil { t.Error(err) } }() go func() { defer wg.Done() if err := s.CancelResource(ctx, v); err != nil { t.Error(err) } }() wg.Wait() if len(s.resourceWaiters) != 0 { t.Fatal("concurrent grant/cancel leaked channel") } } var owners, waiters int s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners) s.db.QueryRow("SELECT COUNT(*) FROM resource_waiters").Scan(&waiters) if owners != 0 || waiters != 0 { t.Fatalf("stranded owners/waiters: %d/%d", owners, waiters) } } func TestResourceCancellationPersistenceFailureRemovesChannel(t *testing.T) { s, create := resourceStore(t) acquireTestResource(t, s, create("X")) v := create("X") if _, _, _, err := s.RequestResource(context.Background(), v); err != nil { t.Fatal(err) } if err := s.Close(); err != nil { t.Fatal(err) } if err := s.CancelResource(context.Background(), v); err == nil { t.Fatal("closed DB cancellation claimed success") } if len(s.resourceWaiters) != 0 { t.Fatal("persistence failure leaked channel") } } func TestResourceRestartQuarantineAndRetention(t *testing.T) { for _, confirmed := range []bool{false, true} { t.Run(fmt.Sprint(confirmed), func(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() a := acquireTestResource(t, s, create("X")) b := create("X") if _, _, _, err := s.RequestResource(ctx, b); err != nil { t.Fatal(err) } if confirmed { if err := s.SetResourceCleanup(ctx, a, true); err != nil { t.Fatal(err) } } path := s.path if err := s.Close(); err != nil { t.Fatal(err) } reopened, err := Open(path) if err != nil { t.Fatal(err) } defer reopened.Close() if err := reopened.RecoverRuns(ctx); err != nil { t.Fatal(err) } if err := reopened.RecoverRuns(ctx); err != nil { t.Fatal(err) } detail, err := reopened.GetRun(ctx, b.RunID) if err != nil || detail.Status != "interrupted" || detail.ResourceWait != nil || len(detail.Steps) != 0 || !strings.Contains(detail.Diagnostic, "resource") { t.Fatal("queued visit replayed/lost diagnostic", detail, err) } exists, err := reopened.HasResourceOwner(ctx, a.RunID) if err != nil || exists == confirmed { t.Fatal("incorrect recovery disposition", exists, err) } report, err := reopened.RetentionReport(ctx, RetentionPolicy{Keep: 0}, "", time.Now()) if err != nil { t.Fatal(err) } protected := false for _, p := range report.Protected { if p.ID == a.RunID { protected = true } } if protected == confirmed { t.Fatal("quarantined evidence not protected") } }) } } func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() owner := acquireTestResource(t, s, create("X")) if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { t.Fatal(err) } waiter := create("X") if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil { t.Fatalf("register waiter: %v", err) } // Simulate a runner disappearing while its durable waiter remains. The // authoritative operation must reject the handoff before audit or cleanup. delete(s.resourceWaiters, waiter) err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"}, }) if err == nil { t.Fatal("reconciled without a live waiter channel") } var cleanup, audits, owners int if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { t.Fatal(err) } if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil { t.Fatal(err) } if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil { t.Fatal(err) } if cleanup != 0 || audits != 0 || owners != 1 { t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners) } } func TestResourceReconcileTransfersAndAudits(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() owner := acquireTestResource(t, s, create("X")) if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { t.Fatal(err) } waiter := create("X") _, ch, _, err := s.RequestResource(ctx, waiter) if err != nil || ch == nil { t.Fatalf("register waiter: %v", err) } if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"}, }); err != nil { t.Fatal(err) } select { case next := <-ch: s.AcceptResource(next) if next.ResourceVisit != waiter { t.Fatalf("wrong handoff: %#v", next.ResourceVisit) } case <-time.After(time.Second): t.Fatal("reconciliation did not wake waiter") } var audits int if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil { t.Fatal(err) } if audits != 1 { t.Fatalf("recovery audit count=%d", audits) } } func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() owner := acquireTestResource(t, s, create("X")) if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { t.Fatal(err) } successor := create("X") _, ch, _, err := s.RequestResource(ctx, successor) if err != nil || ch == nil { t.Fatalf("register successor: %v", err) } request := protocol.ResourceRecoveryRequest{ Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{ CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred", }, } if err := s.ReconcileResource(ctx, request); err != nil { t.Fatal(err) } var successorLease ResourceLease select { case successorLease = <-ch: case <-time.After(time.Second): t.Fatal("reconciliation did not hand off to successor") } if successorLease.ResourceVisit != successor { t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit) } if successorLease.token == "" { t.Fatal("successor capability was empty") } s.AcceptResource(successorLease) // The old request is an idempotent replay after the handoff. It must not // match or release the successor's private capability. if err := s.ReconcileResource(ctx, request); err == nil { t.Fatal("old recovery replay was accepted against successor owner") } inspection, err := s.InspectResource(ctx, owner.Resource) if err != nil { t.Fatal(err) } if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence { t.Fatalf("successor was not preserved: %#v", inspection.Owner) } if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID { t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits) } if len(s.resourceWaiters) != 0 { t.Fatal("successor channel registry leaked") } var owners, waiters, audits int if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil { t.Fatal(err) } if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil { t.Fatal(err) } if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil { t.Fatal(err) } if owners != 1 || waiters != 0 || audits != 1 { t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits) } } func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) { ctx := context.Background() test := func(t *testing.T, setup func(*Store, ResourceLease) int) { s, create := resourceStore(t) owner := acquireTestResource(t, s, create("X")) expectedCleanup := setup(s, owner) err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"}, }) if err == nil { t.Fatal("reconciled unsafe owner") } var owners, audits, cleanup int if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil { t.Fatal(err) } if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil { t.Fatal(err) } if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { t.Fatal(err) } if owners != 1 || audits != 0 || cleanup != expectedCleanup { t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup) } } t.Run("active run", func(t *testing.T) { test(t, func(s *Store, owner ResourceLease) int { if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil { t.Fatal(err) } return 0 }) }) t.Run("active agent", func(t *testing.T) { test(t, func(s *Store, owner ResourceLease) int { if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil { t.Fatal(err) } if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil { t.Fatal(err) } return 0 }) }) t.Run("already confirmed", func(t *testing.T) { test(t, func(s *Store, owner ResourceLease) int { if err := s.SetResourceCleanup(ctx, owner, true); err != nil { t.Fatal(err) } return 1 }) }) } func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) { ctx := context.Background() for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} { t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) { s, create := resourceStore(t) owner := acquireTestResource(t, s, create("X")) err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{ Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence, Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary}, }) if err == nil { t.Fatal("accepted malformed evidence") } var owners, audits, cleanup int if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil { t.Fatal(err) } if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil { t.Fatal(err) } if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil { t.Fatal(err) } if owners != 1 || audits != 0 || cleanup != 0 { t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup) } }) } }