27ad7daecd
Add executable regressions for a forged forwarded loopback claim over a real WebSocket transport and for replaying an old recovery after FIFO successor handoff.\n\nChecked packaged templates and packaged authoring skill references; neither needs updating. The existing per-item documentation note records shared documentation that the manual docs-fold process must update.
531 lines
17 KiB
Go
531 lines
17 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func TestResourceDeclaration(t *testing.T) {
|
|
base := Template([]string{"work"}, true)
|
|
for _, value := range []string{`{"exclusive":"phone-X"}`, `{ "exclusive" : "A_1" }`} {
|
|
graph, err := Parse(base + "```resources\n" + value + "\n```\n")
|
|
if err != nil || graph.Stages[0].Resources == nil {
|
|
t.Fatalf("%s: %v", value, err)
|
|
}
|
|
if _, ok := StageDefinition(graph.Stages[0]).Blocks["resources"]; ok {
|
|
t.Fatal("runner control reached executor")
|
|
}
|
|
encoded, _ := json.Marshal(graph)
|
|
if !strings.Contains(string(encoded), `"resources"`) {
|
|
t.Fatal("resource lost in persisted graph")
|
|
}
|
|
}
|
|
for _, value := range []string{`{}`, `null`, `[]`, `{"exclusive":""}`, `{"exclusive":"a b"}`, `{"exclusive":"a","exclusive":"b"}`, `{"exclusive":"a","extra":1}`, `{"exclusive":["a","b"]}`, fmt.Sprintf(`{"exclusive":%q}`, strings.Repeat("a", 129))} {
|
|
if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil {
|
|
t.Fatalf("accepted %s", value)
|
|
}
|
|
}
|
|
human := "# stage gate\n```prompt\nChoose\n```\n```configs\ntype=human\n```\n```edges\ngo -> #success\n```\n```resources\n{\"exclusive\":\"phone\"}\n```\n"
|
|
if _, err := Parse(human); err == nil {
|
|
t.Fatal("human acquired a resource")
|
|
}
|
|
}
|
|
|
|
func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
s, err := Open(filepath.Join(t.TempDir(), "resources.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
if _, err := s.Submit(ctx, "resource", Template([]string{"work"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
d, _ := s.Get(ctx, items[0].ID, 1)
|
|
count := 0
|
|
return s, func(resource string) ResourceVisit {
|
|
count++
|
|
id, _, err := s.CreateRun(ctx, d, fmt.Sprint(count), "workspace", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return ResourceVisit{Resource: resource, RunID: id, Stage: "work", Sequence: 1}
|
|
}
|
|
}
|
|
|
|
func acquireTestResource(t *testing.T, s *Store, visit ResourceVisit) ResourceLease {
|
|
t.Helper()
|
|
l, ch, _, err := s.RequestResource(context.Background(), visit)
|
|
if err != nil || ch != nil {
|
|
t.Fatalf("acquire: %v %v", ch, err)
|
|
}
|
|
return l
|
|
}
|
|
|
|
func releaseTestResource(t *testing.T, s *Store, lease ResourceLease) {
|
|
t.Helper()
|
|
if err := s.SetResourceCleanup(context.Background(), lease, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.ReleaseResource(context.Background(), lease); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestResourceFIFOOwnershipCancellationAndRegistry(t *testing.T) {
|
|
s, create := resourceStore(t)
|
|
ctx := context.Background()
|
|
a := acquireTestResource(t, s, create("X"))
|
|
duplicate := acquireTestResource(t, s, a.ResourceVisit)
|
|
if duplicate != a {
|
|
t.Fatal("duplicate acquisition changed capability")
|
|
}
|
|
var visits []ResourceVisit
|
|
var channels []<-chan ResourceLease
|
|
for i := 0; i < 4; i++ {
|
|
v := create("X")
|
|
_, ch, wait, err := s.RequestResource(ctx, v)
|
|
if err != nil || ch == nil || wait == nil {
|
|
t.Fatal("missing waiter", err)
|
|
}
|
|
_, again, _, err := s.RequestResource(ctx, v)
|
|
if err != nil || again != ch {
|
|
t.Fatal("duplicate waiter registered", err)
|
|
}
|
|
visits = append(visits, v)
|
|
channels = append(channels, ch)
|
|
}
|
|
if err := s.ReleaseResource(ctx, a); err != nil {
|
|
t.Fatal(err)
|
|
} // no cleanup receipt
|
|
for _, ch := range channels {
|
|
select {
|
|
case <-ch:
|
|
t.Fatal("unconfirmed release granted")
|
|
default:
|
|
}
|
|
}
|
|
if err := s.SetResourceCleanup(ctx, a, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, corrupt := range []func(*ResourceLease){func(l *ResourceLease) { l.RunID++ }, func(l *ResourceLease) { l.Stage = "other" }, func(l *ResourceLease) { l.Sequence++ }, func(l *ResourceLease) { l.token = "wrong" }} {
|
|
bad := a
|
|
corrupt(&bad)
|
|
if err := s.ReleaseResource(ctx, bad); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, ch := range channels {
|
|
select {
|
|
case <-ch:
|
|
t.Fatal("unconfirmed/mismatched release granted")
|
|
default:
|
|
}
|
|
}
|
|
if err := s.CancelResource(ctx, visits[1]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
releaseTestResource(t, s, a)
|
|
for _, i := range []int{0, 2, 3} {
|
|
var grant ResourceLease
|
|
select {
|
|
case grant = <-channels[i]:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("FIFO grant not delivered")
|
|
}
|
|
s.AcceptResource(grant)
|
|
if grant.ResourceVisit != visits[i] {
|
|
t.Fatal("wrong FIFO owner")
|
|
}
|
|
// An old owner's repeated release cannot release the next owner.
|
|
if err := s.ReleaseResource(ctx, a); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, j := range []int{2, 3} {
|
|
if j > i {
|
|
select {
|
|
case <-channels[j]:
|
|
t.Fatal("multiple grants")
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
releaseTestResource(t, s, grant)
|
|
}
|
|
if len(s.resourceWaiters) != 0 {
|
|
t.Fatal("channel registry leaked")
|
|
}
|
|
for i := 0; i < 30; i++ {
|
|
owner := acquireTestResource(t, s, create("X"))
|
|
v := create("X")
|
|
_, _, _, err := s.RequestResource(ctx, v)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
releaseTestResource(t, s, owner) // race window: granted but not accepted
|
|
if err := s.CancelResource(ctx, v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(s.resourceWaiters) != 0 {
|
|
t.Fatal("grant/cancel leaked channel")
|
|
}
|
|
}
|
|
for i := 0; i < 30; i++ {
|
|
owner := acquireTestResource(t, s, create("X"))
|
|
v := create("X")
|
|
if _, _, _, err := s.RequestResource(ctx, v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := s.ReleaseResource(ctx, owner); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
if err := s.CancelResource(ctx, v); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
wg.Wait()
|
|
if len(s.resourceWaiters) != 0 {
|
|
t.Fatal("concurrent grant/cancel leaked channel")
|
|
}
|
|
}
|
|
var owners, waiters int
|
|
s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners)
|
|
s.db.QueryRow("SELECT COUNT(*) FROM resource_waiters").Scan(&waiters)
|
|
if owners != 0 || waiters != 0 {
|
|
t.Fatalf("stranded owners/waiters: %d/%d", owners, waiters)
|
|
}
|
|
}
|
|
|
|
func TestResourceCancellationPersistenceFailureRemovesChannel(t *testing.T) {
|
|
s, create := resourceStore(t)
|
|
acquireTestResource(t, s, create("X"))
|
|
v := create("X")
|
|
if _, _, _, err := s.RequestResource(context.Background(), v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CancelResource(context.Background(), v); err == nil {
|
|
t.Fatal("closed DB cancellation claimed success")
|
|
}
|
|
if len(s.resourceWaiters) != 0 {
|
|
t.Fatal("persistence failure leaked channel")
|
|
}
|
|
}
|
|
|
|
func TestResourceRestartQuarantineAndRetention(t *testing.T) {
|
|
for _, confirmed := range []bool{false, true} {
|
|
t.Run(fmt.Sprint(confirmed), func(t *testing.T) {
|
|
s, create := resourceStore(t)
|
|
ctx := context.Background()
|
|
a := acquireTestResource(t, s, create("X"))
|
|
b := create("X")
|
|
if _, _, _, err := s.RequestResource(ctx, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if confirmed {
|
|
if err := s.SetResourceCleanup(ctx, a, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
path := s.path
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reopened, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer reopened.Close()
|
|
if err := reopened.RecoverRuns(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := reopened.RecoverRuns(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
detail, err := reopened.GetRun(ctx, b.RunID)
|
|
if err != nil || detail.Status != "interrupted" || detail.ResourceWait != nil || len(detail.Steps) != 0 || !strings.Contains(detail.Diagnostic, "resource") {
|
|
t.Fatal("queued visit replayed/lost diagnostic", detail, err)
|
|
}
|
|
exists, err := reopened.HasResourceOwner(ctx, a.RunID)
|
|
if err != nil || exists == confirmed {
|
|
t.Fatal("incorrect recovery disposition", exists, err)
|
|
}
|
|
report, err := reopened.RetentionReport(ctx, RetentionPolicy{Keep: 0}, "", time.Now())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
protected := false
|
|
for _, p := range report.Protected {
|
|
if p.ID == a.RunID {
|
|
protected = true
|
|
}
|
|
}
|
|
if protected == confirmed {
|
|
t.Fatal("quarantined evidence not protected")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResourceReconcileMissingWaiterPreservesQuarantine(t *testing.T) {
|
|
s, create := resourceStore(t)
|
|
ctx := context.Background()
|
|
owner := acquireTestResource(t, s, create("X"))
|
|
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waiter := create("X")
|
|
if _, ch, _, err := s.RequestResource(ctx, waiter); err != nil || ch == nil {
|
|
t.Fatalf("register waiter: %v", err)
|
|
}
|
|
// Simulate a runner disappearing while its durable waiter remains. The
|
|
// authoritative operation must reject the handoff before audit or cleanup.
|
|
delete(s.resourceWaiters, waiter)
|
|
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
|
|
Resource: owner.Resource,
|
|
RunID: owner.RunID,
|
|
Stage: owner.Stage,
|
|
Sequence: owner.Sequence,
|
|
Actor: "operator",
|
|
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
|
|
})
|
|
if err == nil {
|
|
t.Fatal("reconciled without a live waiter channel")
|
|
}
|
|
var cleanup, audits, owners int
|
|
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cleanup != 0 || audits != 0 || owners != 1 {
|
|
t.Fatalf("failed recovery changed state: cleanup=%d audits=%d owners=%d", cleanup, audits, owners)
|
|
}
|
|
}
|
|
|
|
func TestResourceReconcileTransfersAndAudits(t *testing.T) {
|
|
s, create := resourceStore(t)
|
|
ctx := context.Background()
|
|
owner := acquireTestResource(t, s, create("X"))
|
|
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waiter := create("X")
|
|
_, ch, _, err := s.RequestResource(ctx, waiter)
|
|
if err != nil || ch == nil {
|
|
t.Fatalf("register waiter: %v", err)
|
|
}
|
|
if err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
|
|
Resource: owner.Resource,
|
|
RunID: owner.RunID,
|
|
Stage: owner.Stage,
|
|
Sequence: owner.Sequence,
|
|
Actor: "operator",
|
|
Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, NoMutatingAction: true, Summary: "inspection confirmed no mutating action occurred"},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case next := <-ch:
|
|
s.AcceptResource(next)
|
|
if next.ResourceVisit != waiter {
|
|
t.Fatalf("wrong handoff: %#v", next.ResourceVisit)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("reconciliation did not wake waiter")
|
|
}
|
|
var audits int
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=? AND result='reconciled'", owner.Resource).Scan(&audits); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if audits != 1 {
|
|
t.Fatalf("recovery audit count=%d", audits)
|
|
}
|
|
}
|
|
|
|
func TestResourceReconcileReplayAfterSuccessorHandoff(t *testing.T) {
|
|
s, create := resourceStore(t)
|
|
ctx := context.Background()
|
|
owner := acquireTestResource(t, s, create("X"))
|
|
if _, err := s.db.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
successor := create("X")
|
|
_, ch, _, err := s.RequestResource(ctx, successor)
|
|
if err != nil || ch == nil {
|
|
t.Fatalf("register successor: %v", err)
|
|
}
|
|
request := protocol.ResourceRecoveryRequest{
|
|
Resource: owner.Resource,
|
|
RunID: owner.RunID,
|
|
Stage: owner.Stage,
|
|
Sequence: owner.Sequence,
|
|
Actor: "operator",
|
|
Evidence: &protocol.ResourceRecoveryEvidence{
|
|
CleanupConfirmed: true,
|
|
NoMutatingAction: true,
|
|
Summary: "inspection confirmed no mutating action occurred",
|
|
},
|
|
}
|
|
if err := s.ReconcileResource(ctx, request); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var successorLease ResourceLease
|
|
select {
|
|
case successorLease = <-ch:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("reconciliation did not hand off to successor")
|
|
}
|
|
if successorLease.ResourceVisit != successor {
|
|
t.Fatalf("wrong successor visit: %#v", successorLease.ResourceVisit)
|
|
}
|
|
if successorLease.token == "" {
|
|
t.Fatal("successor capability was empty")
|
|
}
|
|
s.AcceptResource(successorLease)
|
|
|
|
// The old request is an idempotent replay after the handoff. It must not
|
|
// match or release the successor's private capability.
|
|
if err := s.ReconcileResource(ctx, request); err == nil {
|
|
t.Fatal("old recovery replay was accepted against successor owner")
|
|
}
|
|
inspection, err := s.InspectResource(ctx, owner.Resource)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if inspection.Owner == nil || inspection.Owner.RunID != successor.RunID || inspection.Owner.Stage != successor.Stage || inspection.Owner.Sequence != successor.Sequence {
|
|
t.Fatalf("successor was not preserved: %#v", inspection.Owner)
|
|
}
|
|
if len(inspection.Waiters) != 0 || len(inspection.Audits) != 1 || inspection.Audits[0].RunID != owner.RunID {
|
|
t.Fatalf("replay changed handoff state: waiters=%d audits=%#v", len(inspection.Waiters), inspection.Audits)
|
|
}
|
|
if len(s.resourceWaiters) != 0 {
|
|
t.Fatal("successor channel registry leaked")
|
|
}
|
|
var owners, waiters, audits int
|
|
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&owners); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_waiters WHERE resource_key=?", owner.Resource).Scan(&waiters); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_recovery_audit WHERE resource_key=?", owner.Resource).Scan(&audits); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if owners != 1 || waiters != 0 || audits != 1 {
|
|
t.Fatalf("unexpected post-replay counts: owners=%d waiters=%d audits=%d", owners, waiters, audits)
|
|
}
|
|
}
|
|
|
|
func TestResourceReconcileRequiresQuarantinedInactiveOwner(t *testing.T) {
|
|
ctx := context.Background()
|
|
test := func(t *testing.T, setup func(*Store, ResourceLease) int) {
|
|
s, create := resourceStore(t)
|
|
owner := acquireTestResource(t, s, create("X"))
|
|
expectedCleanup := setup(s, owner)
|
|
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
|
|
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
|
|
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: "verified"},
|
|
})
|
|
if err == nil {
|
|
t.Fatal("reconciled unsafe owner")
|
|
}
|
|
var owners, audits, cleanup int
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if owners != 1 || audits != 0 || cleanup != expectedCleanup {
|
|
t.Fatalf("unsafe recovery changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
|
|
}
|
|
}
|
|
t.Run("active run", func(t *testing.T) {
|
|
test(t, func(s *Store, owner ResourceLease) int {
|
|
if _, err := s.db.Exec("UPDATE pipeline_runs SET status=? WHERE id=?", protocol.RunStatusWaiting, owner.RunID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return 0
|
|
})
|
|
})
|
|
t.Run("active agent", func(t *testing.T) {
|
|
test(t, func(s *Store, owner ResourceLease) int {
|
|
if _, err := s.db.Exec("UPDATE pipeline_runs SET status='interrupted' WHERE id=?", owner.RunID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.db.Exec("INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?, '{}')", owner.RunID, owner.Sequence, owner.Stage, "running", 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return 0
|
|
})
|
|
})
|
|
t.Run("already confirmed", func(t *testing.T) {
|
|
test(t, func(s *Store, owner ResourceLease) int {
|
|
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return 1
|
|
})
|
|
})
|
|
}
|
|
|
|
func TestResourceReconcileRejectsMalformedEvidenceBeforeMutation(t *testing.T) {
|
|
ctx := context.Background()
|
|
for _, summary := range []string{"", "true", "FALSE", "null", "verified\x00cleanup", strings.Repeat("x", maxRecoveryText+1)} {
|
|
t.Run(fmt.Sprintf("%q", summary), func(t *testing.T) {
|
|
s, create := resourceStore(t)
|
|
owner := acquireTestResource(t, s, create("X"))
|
|
err := s.ReconcileResource(ctx, protocol.ResourceRecoveryRequest{
|
|
Resource: owner.Resource, RunID: owner.RunID, Stage: owner.Stage, Sequence: owner.Sequence,
|
|
Actor: "operator", Evidence: &protocol.ResourceRecoveryEvidence{CleanupConfirmed: true, Summary: summary},
|
|
})
|
|
if err == nil {
|
|
t.Fatal("accepted malformed evidence")
|
|
}
|
|
var owners, audits, cleanup int
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_owners").Scan(&owners); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM resource_recovery_audit").Scan(&audits); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.QueryRow("SELECT cleanup_confirmed FROM resource_owners WHERE resource_key=?", owner.Resource).Scan(&cleanup); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if owners != 1 || audits != 0 || cleanup != 0 {
|
|
t.Fatalf("invalid evidence changed state: owners=%d audits=%d cleanup=%d", owners, audits, cleanup)
|
|
}
|
|
})
|
|
}
|
|
}
|