diff --git a/docs/notes/android-target-pool-allocation.md b/docs/notes/android-target-pool-allocation.md new file mode 100644 index 00000000..4e1ceb4c --- /dev/null +++ b/docs/notes/android-target-pool-allocation.md @@ -0,0 +1,25 @@ +# Android target pool allocation — 2026-10-18, phase one not activated + +This item adds a reusable store contract only. Pool declarations are not +activated in live jobs, runner handoff, server configuration, Android maps, +shared documentation, or deployment. + +`Store.ConfigureResourcePool(ctx, protocol.ResourcePool)` replaces a bounded +pool declaration. Each member has a unique 1–128 byte resource key, an enabled +flag, quarantine state, and at most 16 bounded ASCII-keyed metadata values +(each value is at most 128 bytes). A pool has 1–32 explicit members. Invalid, +empty, duplicate, unknown, or malformed declarations are rejected. Updating a +pool never transfers a live owner; queued requests use the new declaration on +the next eligible release. + +`Store.RequestResourcePool(ctx, pool, visit)` atomically returns a lease for one +free eligible concrete member, or a FIFO wait channel. The lease's `Target` +contains only the concrete member and bounded non-secret metadata. Capability +tokens remain private to the store and are never protocol/model metadata. +Direct concrete-resource leases and pool leases share the owner table, so they +cannot overlap. Disabled or quarantined members are skipped; if none are +eligible, the request queues without inference. Release, cancellation, stale +credentials, recovery, and restart retain the existing no-expiry semantics. + +This is scheduler coordination only: no device probing, emulator creation, +credentials, activation, or live job changes are included. diff --git a/internal/pipeline/resources.go b/internal/pipeline/resources.go index 5d4907a8..47cef572 100644 --- a/internal/pipeline/resources.go +++ b/internal/pipeline/resources.go @@ -193,7 +193,189 @@ type ResourceVisit struct { // or serializable, and must never be put in model input or public wire metadata. type ResourceLease struct { ResourceVisit - token string + Target *protocol.ResourceTarget + pool string + token string +} + +const maxPoolMembers = 32 +const maxTargetMetadata = 16 +const maxTargetValue = 128 + +func validateTargetMetadata(metadata map[string]string) error { + if len(metadata) > maxTargetMetadata { + return fmt.Errorf("too many target metadata fields") + } + for k, v := range metadata { + if !identifier.MatchString(k) || len(v) > maxTargetValue || !utf8.ValidString(v) { + return fmt.Errorf("invalid target metadata") + } + } + return nil +} + +func validateResourcePool(name string, members []protocol.ResourcePoolMember) error { + if !ValidResourceKey(name) || len(members) == 0 || len(members) > maxPoolMembers { + return fmt.Errorf("invalid resource pool") + } + seen := make(map[string]bool, len(members)) + for i := range members { + member := &members[i] + if !ValidResourceKey(member.Member) || seen[member.Member] { + return fmt.Errorf("invalid pool member") + } + if err := validateTargetMetadata(member.Metadata); err != nil { + return err + } + seen[member.Member] = true + } + return nil +} + +// ConfigureResourcePool replaces a pool declaration atomically. Existing +// owners keep their concrete member; queued requests are evaluated against the +// new declaration on the next release and are never silently transferred. +func (s *Store) ConfigureResourcePool(ctx context.Context, pool protocol.ResourcePool) error { + if err := validateResourcePool(pool.Name, pool.Members); err != nil { + return err + } + encoded, err := json.Marshal(pool.Members) + if err != nil { + return err + } + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + _, err = s.db.ExecContext(ctx, `INSERT INTO resource_pools(name,members_json) VALUES(?,?) ON CONFLICT(name) DO UPDATE SET members_json=excluded.members_json,updated_at=CURRENT_TIMESTAMP`, pool.Name, string(encoded)) + return err +} + +func (s *Store) resourcePool(ctx context.Context, tx interface { + QueryRowContext(context.Context, string, ...any) *sql.Row +}, name string) ([]protocol.ResourcePoolMember, error) { + var encoded string + if err := tx.QueryRowContext(ctx, "SELECT members_json FROM resource_pools WHERE name=?", name).Scan(&encoded); err != nil { + return nil, err + } + var members []protocol.ResourcePoolMember + if err := json.Unmarshal([]byte(encoded), &members); err != nil { + return nil, fmt.Errorf("invalid persisted pool: %w", err) + } + if err := validateResourcePool(name, members); err != nil { + return nil, fmt.Errorf("invalid persisted pool: %w", err) + } + return members, nil +} + +func poolTarget(member protocol.ResourcePoolMember) *protocol.ResourceTarget { + metadata := make(map[string]string, len(member.Metadata)) + for k, v := range member.Metadata { + metadata[k] = v + } + return &protocol.ResourceTarget{Member: member.Member, Metadata: metadata} +} + +// RequestResourcePool selects one free eligible concrete member while holding +// the same authority as direct resource requests. +func (s *Store) RequestResourcePool(ctx context.Context, pool string, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) { + if !ValidResourceKey(pool) || visit.Resource != pool || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) { + return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource pool visit") + } + s.resourceMu.Lock() + defer s.resourceMu.Unlock() + if ch := s.resourceWaiters[visit]; ch != nil { + wait, err := s.resourceWait(ctx, visit.RunID) + return ResourceLease{}, ch, wait, err + } + credential := make([]byte, 32) + if _, err := rand.Read(credential); err != nil { + return ResourceLease{}, nil, nil, err + } + lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)} + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ResourceLease{}, nil, nil, err + } + defer tx.Rollback() + var status string + if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil { + return ResourceLease{}, nil, nil, err + } + if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning { + return ResourceLease{}, nil, nil, fmt.Errorf("run is not active") + } + var existing ResourceLease + var existingPool string + err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool) + if err == nil { + if existing.RunID != visit.RunID || existing.Stage != visit.Stage || existing.Sequence != visit.Sequence { + return ResourceLease{}, nil, nil, fmt.Errorf("resource owner visit is stale") + } + if existingPool != pool { + return ResourceLease{}, nil, nil, fmt.Errorf("resource owner pool visit is stale") + } + members, poolErr := s.resourcePool(ctx, tx, pool) + if poolErr != nil { + return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", poolErr) + } + for _, member := range members { + if member.Member == existing.Resource { + existing.ResourceVisit = visit + existing.pool = pool + existing.Target = poolTarget(member) + return existing, nil, nil, nil + } + } + return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit") + } + if err != sql.ErrNoRows { + return ResourceLease{}, nil, nil, err + } + members, err := s.resourcePool(ctx, tx, pool) + if err != nil { + return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", err) + } + for _, member := range members { + if !member.Enabled || member.Quarantined { + continue + } + var count int + if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); err != nil { + return ResourceLease{}, nil, nil, err + } + if count != 0 { + continue + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, member.Member, visit.RunID, visit.Stage, visit.Sequence, lease.token, pool); err != nil { + return ResourceLease{}, nil, nil, err + } + lease.Resource = member.Member + lease.pool = pool + lease.Target = poolTarget(member) + if err := tx.Commit(); err != nil { + return ResourceLease{}, nil, nil, err + } + return lease, nil, nil, nil + } + wait := &protocol.ResourceWait{Resource: pool, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)} + encoded, err := json.Marshal(wait) + if err != nil { + return ResourceLease{}, nil, nil, err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,pool_name,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?,?)`, pool, pool, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil { + return ResourceLease{}, nil, nil, err + } + if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil { + return ResourceLease{}, nil, nil, err + } + if err := tx.Commit(); err != nil { + return ResourceLease{}, nil, nil, err + } + if s.resourceWaiters == nil { + s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease) + } + ch := make(chan ResourceLease, 1) + s.resourceWaiters[visit] = ch + return ResourceLease{}, ch, wait, nil } // RequestResource acquires immediately or registers one FIFO waiter/channel. The @@ -226,8 +408,12 @@ func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (Resou return ResourceLease{}, nil, nil, fmt.Errorf("run is not active") } var existing ResourceLease - err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token) + var existingPool string + err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool) if err == nil { + if existingPool != "" { + return ResourceLease{}, nil, nil, fmt.Errorf("run already owns a pooled resource visit") + } if existing.ResourceVisit != visit { return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit") } @@ -236,8 +422,8 @@ func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (Resou if err != sql.ErrNoRows { return ResourceLease{}, nil, nil, err } - result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) -VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token) + result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) +VALUES(?,?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, "") if err != nil { return ResourceLease{}, nil, nil, err } @@ -349,11 +535,18 @@ func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner) // SetResourceCleanup records positive quiescence durably before release. False // quarantines the owner; neither age nor another run can remove the lease. func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error { + if lease.pool != "" && (lease.Target == nil || !ValidResourceKey(lease.Target.Member)) { + return fmt.Errorf("invalid pooled resource lease") + } diagnostic := "" if !confirmed { diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation" } - return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)) + resource := lease.Resource + if lease.pool != "" { + resource = lease.Target.Member + } + return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, resource, lease.RunID, lease.Stage, lease.Sequence, lease.token)) } // ReleaseResource requires the complete capability AND positive cleanup evidence. @@ -370,6 +563,14 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) return err } defer tx.Rollback() + ownerResource := lease.Resource + if lease.pool != "" { + if lease.Target == nil || !ValidResourceKey(lease.Target.Member) { + return fmt.Errorf("invalid pooled resource lease") + } + ownerResource = lease.Target.Member + } + lease.Resource = ownerResource n, err := deleteResourceOwner(ctx, tx, lease) if err != nil { return err @@ -377,7 +578,7 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) if n == 0 { return nil } - next, err := s.handoffResourceTx(ctx, tx, lease.Resource) + next, err := s.handoffResourceTx(ctx, tx, ownerResource) if err != nil { return err } @@ -396,25 +597,86 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease) func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) { var next ResourceLease next.Resource = resource - err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token) - if err == sql.ErrNoRows { - return ResourceLease{}, nil - } + rows, err := tx.QueryContext(ctx, `SELECT queue_order,resource_key,pool_name,run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? OR pool_name<>'' ORDER BY queue_order`, resource) if err != nil { return ResourceLease{}, err } - if s.resourceWaiters[next.ResourceVisit] == nil { - return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff") + defer rows.Close() + var selected *protocol.ResourceTarget + var queue int64 + var waiterResource string + for rows.Next() { + var candidate ResourceLease + var candidatePool string + if err := rows.Scan(&queue, &waiterResource, &candidatePool, &candidate.RunID, &candidate.Stage, &candidate.Sequence, &candidate.token); err != nil { + return ResourceLease{}, err + } + if candidatePool != "" { + candidate.Resource = candidatePool + candidate.pool = candidatePool + } else { + candidate.Resource = resource + } + if candidatePool == "" { + if waiterResource != resource { + continue + } + selected = &protocol.ResourceTarget{Member: resource} + } else { + members, e := s.resourcePool(ctx, tx, candidatePool) + if e != nil { + continue + } + found := false + for _, member := range members { + if member.Member != resource || !member.Enabled || member.Quarantined { + continue + } + var count int + if e = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); e != nil { + return ResourceLease{}, e + } + if count == 0 { + selected, found = poolTarget(member), true + } + break + } + if !found { + continue + } + } + if s.resourceWaiters[candidate.ResourceVisit] == nil { + if candidatePool == "" { + return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff") + } + continue + } + next = candidate + next.Target = selected + break } - if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil { + if err := rows.Err(); err != nil { return ResourceLease{}, err } - if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil { + if next.token == "" { + return ResourceLease{}, nil + } + ownerResource := next.Resource + if next.pool != "" { + ownerResource = next.Target.Member + } + if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, ownerResource, next.RunID, next.Stage, next.Sequence, next.token, next.pool); err != nil { + return ResourceLease{}, err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE queue_order=?`, queue); err != nil { return ResourceLease{}, err } if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil { return ResourceLease{}, err } + if next.pool == "" { + next.Resource = ownerResource + } return next, nil } diff --git a/internal/pipeline/resources_test.go b/internal/pipeline/resources_test.go index c9a1c4b3..ac4980d7 100644 --- a/internal/pipeline/resources_test.go +++ b/internal/pipeline/resources_test.go @@ -39,6 +39,206 @@ func TestResourceDeclaration(t *testing.T) { } } +func TestResourcePoolAllocatesConcreteMembersAndExcludesDirectOwner(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "android-targets", Members: []protocol.ResourcePoolMember{ + {Member: "phone", Enabled: true, Metadata: map[string]string{"kind": "phone"}}, + {Member: "emulator", Enabled: true, Metadata: map[string]string{"kind": "emulator"}}, + }}); err != nil { + t.Fatal(err) + } + phone := acquireTestResource(t, s, create("phone")) + visit := create("android-targets") + lease, ch, _, err := s.RequestResourcePool(ctx, "android-targets", visit) + if err != nil || ch != nil || lease.Target == nil || lease.Target.Member != "emulator" { + t.Fatalf("pool selected %#v, channel=%v, err=%v", lease.Target, ch, err) + } + if lease.Target.Metadata["kind"] != "emulator" { + t.Fatal("missing bounded target metadata") + } + releaseTestResource(t, s, lease) + releaseTestResource(t, s, phone) +} + +func TestResourcePoolFIFOWakeAndQuarantineIsolation(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ + {Member: "held", Enabled: true}, {Member: "free", Enabled: true}, {Member: "bad", Enabled: false, Quarantined: true}, + }}); err != nil { + t.Fatal(err) + } + held := acquireTestResource(t, s, create("held")) + first := create("targets") + firstLease, firstCh, _, err := s.RequestResourcePool(ctx, "targets", first) + if err != nil || firstCh != nil || firstLease.Target == nil { + t.Fatalf("expected immediate free member: %v", err) + } + // The second request waits once both concrete members are held. + second := create("targets") + _, secondCh, _, err := s.RequestResourcePool(ctx, "targets", second) + if err != nil || secondCh == nil { + t.Fatalf("expected pool waiter: %v", err) + } + releaseTestResource(t, s, firstLease) + select { + case next := <-secondCh: + if next.Target == nil || next.Target.Member != "free" { + t.Fatalf("wrong handoff: %#v", next.Target) + } + s.AcceptResource(next) + releaseTestResource(t, s, next) + case <-time.After(time.Second): + t.Fatal("pool waiter not woken") + } + releaseTestResource(t, s, held) +} + +func TestResourcePoolRejectsStaleVisitForExistingOwner(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ + {Member: "phone", Enabled: true}, + }}); err != nil { + t.Fatal(err) + } + owner := acquireTestResource(t, s, create("phone")) + stale := owner.ResourceVisit + stale.Resource = "targets" + stale.Sequence++ + if _, ch, _, err := s.RequestResourcePool(ctx, "targets", stale); err == nil || ch != nil { + t.Fatalf("stale pool visit was accepted: channel=%v err=%v", ch, err) + } + if err := s.SetResourceCleanup(ctx, owner, true); err != nil { + t.Fatal(err) + } + if err := s.ReleaseResource(ctx, owner); err != nil { + t.Fatal(err) + } +} + +func TestResourcePoolAndDirectVisitsCannotBeReinterpreted(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true}}}); err != nil { + t.Fatal(err) + } + pooled := create("targets") + lease, ch, _, err := s.RequestResourcePool(ctx, "targets", pooled) + if err != nil || ch != nil { + t.Fatalf("pool acquire: %v %v", ch, err) + } + if _, ch, _, err := s.RequestResource(ctx, ResourceVisit{Resource: "phone", RunID: pooled.RunID, Stage: pooled.Stage, Sequence: pooled.Sequence}); err == nil || ch != nil { + t.Fatalf("pooled owner was accepted as direct lease: channel=%v err=%v", ch, err) + } + releaseTestResource(t, s, lease) +} + +func TestResourcePoolConfigurationAndOwnerSurviveReopen(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ + {Member: "phone", Enabled: true, Metadata: map[string]string{"kind": "phone"}}, + }}); err != nil { + t.Fatal(err) + } + visit := create("targets") + owner, ch, _, err := s.RequestResourcePool(ctx, "targets", visit) + if err != nil || ch != nil { + t.Fatalf("pool acquire: %v %v", ch, err) + } + path := s.path + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + recovered, ch, _, err := reopened.RequestResourcePool(ctx, "targets", visit) + if err != nil || ch != nil || recovered.Target == nil || recovered.Target.Member != "phone" || recovered.Target.Metadata["kind"] != "phone" { + t.Fatalf("reopened pool owner not recovered: target=%#v channel=%v err=%v", recovered.Target, ch, err) + } + if err := reopened.SetResourceCleanup(ctx, recovered, true); err != nil { + t.Fatal(err) + } + if err := reopened.ReleaseResource(ctx, recovered); err != nil { + t.Fatal(err) + } + _ = owner +} + +func TestResourcePoolAllocatesDistinctMembersConcurrently(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ + {Member: "phone", Enabled: true}, {Member: "emulator", Enabled: true}, + }}); err != nil { + t.Fatal(err) + } + visits := []ResourceVisit{create("targets"), create("targets")} + leases := make(chan ResourceLease, len(visits)) + errs := make(chan error, len(visits)) + var wg sync.WaitGroup + for _, visit := range visits { + wg.Add(1) + go func(v ResourceVisit) { + defer wg.Done() + lease, ch, _, err := s.RequestResourcePool(ctx, "targets", v) + if err != nil { + errs <- err + return + } + if ch != nil { + select { + case lease = <-ch: + case <-time.After(time.Second): + errs <- fmt.Errorf("pool request remained queued") + return + } + s.AcceptResource(lease) + } + leases <- lease + }(visit) + } + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + got := map[string]bool{} + for range visits { + lease := <-leases + if lease.Target == nil || got[lease.Target.Member] { + t.Fatalf("members were not distinct: %#v", lease.Target) + } + got[lease.Target.Member] = true + releaseTestResource(t, s, lease) + } +} + +func TestResourcePoolRejectsInvalidDeclarations(t *testing.T) { + s, _ := resourceStore(t) + ctx := context.Background() + cases := []protocol.ResourcePool{ + {Name: "missing-members"}, + {Name: "duplicate", Members: []protocol.ResourcePoolMember{{Member: "phone"}, {Member: "phone"}}}, + {Name: "bad-member", Members: []protocol.ResourcePoolMember{{Member: "not valid"}}}, + {Name: "bad-metadata", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{"bad key": "x"}}}}, + } + for _, pool := range cases { + if err := s.ConfigureResourcePool(ctx, pool); err == nil { + t.Fatalf("accepted invalid pool %#v", pool) + } + } + visit := ResourceVisit{Resource: "unknown", RunID: 1, Stage: "work", Sequence: 1} + if _, _, _, err := s.RequestResourcePool(ctx, "unknown", visit); err == nil { + t.Fatal("accepted unknown pool") + } +} + func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) { t.Helper() ctx := context.Background() diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index ff766442..55af004c 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -58,7 +58,7 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints ( source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); CREATE TABLE IF NOT EXISTS resource_owners ( resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), -stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, + stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, pool_name TEXT NOT NULL DEFAULT '', cleanup_confirmed INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '', acquired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); CREATE INDEX IF NOT EXISTS resource_owner_run ON resource_owners(run_id); @@ -71,6 +71,17 @@ CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,q if err != nil { return err } + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_pools ( + name TEXT PRIMARY KEY, members_json TEXT NOT NULL, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +);`); err != nil { + return err + } + if err := s.addColumnIfMissing("resource_waiters", "pool_name", "TEXT NOT NULL DEFAULT ''"); err != nil { + return err + } + if err := s.addColumnIfMissing("resource_owners", "pool_name", "TEXT NOT NULL DEFAULT ''"); err != nil { + return err + } if err := s.migrateRunTitles(); err != nil { return err } diff --git a/internal/protocol/resources.go b/internal/protocol/resources.go index 4f9c3c53..8f31867e 100644 --- a/internal/protocol/resources.go +++ b/internal/protocol/resources.go @@ -6,6 +6,25 @@ type ResourceInspection struct { Waiters []ResourceWaiter `json:"waiters,omitempty"` Audits []ResourceRecoveryAudit `json:"audits,omitempty"` } + +// ResourceTarget is bounded, non-secret metadata for the concrete member that +// was selected. Credentials and capabilities are intentionally absent. +type ResourceTarget struct { + Member string `json:"member"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +type ResourcePoolMember struct { + Member string `json:"member"` + Metadata map[string]string `json:"metadata,omitempty"` + Enabled bool `json:"enabled"` + Quarantined bool `json:"quarantined,omitempty"` +} + +type ResourcePool struct { + Name string `json:"name"` + Members []ResourcePoolMember `json:"members"` +} type ResourceOwner struct { RunID int64 `json:"run_id"` Stage string `json:"stage"`