feat: add pooled resource allocation contract
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
# Android target pool allocation — 2026-10-18, phase one not activated
|
||||
|
||||
This item adds a reusable store contract only. Pool declarations are not
|
||||
activated in live jobs, runner handoff, server configuration, Android maps,
|
||||
shared documentation, or deployment.
|
||||
|
||||
`Store.ConfigureResourcePool(ctx, protocol.ResourcePool)` replaces a bounded
|
||||
pool declaration. Each member has a unique 1–128 byte resource key, an enabled
|
||||
flag, quarantine state, and at most 16 bounded ASCII-keyed metadata values
|
||||
(each value is at most 128 bytes). A pool has 1–32 explicit members. Invalid,
|
||||
empty, duplicate, unknown, or malformed declarations are rejected. Updating a
|
||||
pool never transfers a live owner; queued requests use the new declaration on
|
||||
the next eligible release.
|
||||
|
||||
`Store.RequestResourcePool(ctx, pool, visit)` atomically returns a lease for one
|
||||
free eligible concrete member, or a FIFO wait channel. The lease's `Target`
|
||||
contains only the concrete member and bounded non-secret metadata. Capability
|
||||
tokens remain private to the store and are never protocol/model metadata.
|
||||
Direct concrete-resource leases and pool leases share the owner table, so they
|
||||
cannot overlap. Disabled or quarantined members are skipped; if none are
|
||||
eligible, the request queues without inference. Release, cancellation, stale
|
||||
credentials, recovery, and restart retain the existing no-expiry semantics.
|
||||
|
||||
This is scheduler coordination only: no device probing, emulator creation,
|
||||
credentials, activation, or live job changes are included.
|
||||
+276
-14
@@ -193,7 +193,189 @@ type ResourceVisit struct {
|
||||
// or serializable, and must never be put in model input or public wire metadata.
|
||||
type ResourceLease struct {
|
||||
ResourceVisit
|
||||
token string
|
||||
Target *protocol.ResourceTarget
|
||||
pool string
|
||||
token string
|
||||
}
|
||||
|
||||
const maxPoolMembers = 32
|
||||
const maxTargetMetadata = 16
|
||||
const maxTargetValue = 128
|
||||
|
||||
func validateTargetMetadata(metadata map[string]string) error {
|
||||
if len(metadata) > maxTargetMetadata {
|
||||
return fmt.Errorf("too many target metadata fields")
|
||||
}
|
||||
for k, v := range metadata {
|
||||
if !identifier.MatchString(k) || len(v) > maxTargetValue || !utf8.ValidString(v) {
|
||||
return fmt.Errorf("invalid target metadata")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateResourcePool(name string, members []protocol.ResourcePoolMember) error {
|
||||
if !ValidResourceKey(name) || len(members) == 0 || len(members) > maxPoolMembers {
|
||||
return fmt.Errorf("invalid resource pool")
|
||||
}
|
||||
seen := make(map[string]bool, len(members))
|
||||
for i := range members {
|
||||
member := &members[i]
|
||||
if !ValidResourceKey(member.Member) || seen[member.Member] {
|
||||
return fmt.Errorf("invalid pool member")
|
||||
}
|
||||
if err := validateTargetMetadata(member.Metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
seen[member.Member] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConfigureResourcePool replaces a pool declaration atomically. Existing
|
||||
// owners keep their concrete member; queued requests are evaluated against the
|
||||
// new declaration on the next release and are never silently transferred.
|
||||
func (s *Store) ConfigureResourcePool(ctx context.Context, pool protocol.ResourcePool) error {
|
||||
if err := validateResourcePool(pool.Name, pool.Members); err != nil {
|
||||
return err
|
||||
}
|
||||
encoded, err := json.Marshal(pool.Members)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.resourceMu.Lock()
|
||||
defer s.resourceMu.Unlock()
|
||||
_, err = s.db.ExecContext(ctx, `INSERT INTO resource_pools(name,members_json) VALUES(?,?) ON CONFLICT(name) DO UPDATE SET members_json=excluded.members_json,updated_at=CURRENT_TIMESTAMP`, pool.Name, string(encoded))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) resourcePool(ctx context.Context, tx interface {
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}, name string) ([]protocol.ResourcePoolMember, error) {
|
||||
var encoded string
|
||||
if err := tx.QueryRowContext(ctx, "SELECT members_json FROM resource_pools WHERE name=?", name).Scan(&encoded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var members []protocol.ResourcePoolMember
|
||||
if err := json.Unmarshal([]byte(encoded), &members); err != nil {
|
||||
return nil, fmt.Errorf("invalid persisted pool: %w", err)
|
||||
}
|
||||
if err := validateResourcePool(name, members); err != nil {
|
||||
return nil, fmt.Errorf("invalid persisted pool: %w", err)
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
func poolTarget(member protocol.ResourcePoolMember) *protocol.ResourceTarget {
|
||||
metadata := make(map[string]string, len(member.Metadata))
|
||||
for k, v := range member.Metadata {
|
||||
metadata[k] = v
|
||||
}
|
||||
return &protocol.ResourceTarget{Member: member.Member, Metadata: metadata}
|
||||
}
|
||||
|
||||
// RequestResourcePool selects one free eligible concrete member while holding
|
||||
// the same authority as direct resource requests.
|
||||
func (s *Store) RequestResourcePool(ctx context.Context, pool string, visit ResourceVisit) (ResourceLease, <-chan ResourceLease, *protocol.ResourceWait, error) {
|
||||
if !ValidResourceKey(pool) || visit.Resource != pool || visit.RunID <= 0 || visit.Sequence <= 0 || !identifier.MatchString(visit.Stage) {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("invalid resource pool visit")
|
||||
}
|
||||
s.resourceMu.Lock()
|
||||
defer s.resourceMu.Unlock()
|
||||
if ch := s.resourceWaiters[visit]; ch != nil {
|
||||
wait, err := s.resourceWait(ctx, visit.RunID)
|
||||
return ResourceLease{}, ch, wait, err
|
||||
}
|
||||
credential := make([]byte, 32)
|
||||
if _, err := rand.Read(credential); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
lease := ResourceLease{ResourceVisit: visit, token: hex.EncodeToString(credential)}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var status string
|
||||
if err := tx.QueryRowContext(ctx, "SELECT status FROM pipeline_runs WHERE id=?", visit.RunID).Scan(&status); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if status != protocol.RunStatusQueued && status != protocol.RunStatusRunning {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
|
||||
}
|
||||
var existing ResourceLease
|
||||
var existingPool string
|
||||
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool)
|
||||
if err == nil {
|
||||
if existing.RunID != visit.RunID || existing.Stage != visit.Stage || existing.Sequence != visit.Sequence {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("resource owner visit is stale")
|
||||
}
|
||||
if existingPool != pool {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("resource owner pool visit is stale")
|
||||
}
|
||||
members, poolErr := s.resourcePool(ctx, tx, pool)
|
||||
if poolErr != nil {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", poolErr)
|
||||
}
|
||||
for _, member := range members {
|
||||
if member.Member == existing.Resource {
|
||||
existing.ResourceVisit = visit
|
||||
existing.pool = pool
|
||||
existing.Target = poolTarget(member)
|
||||
return existing, nil, nil, nil
|
||||
}
|
||||
}
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
members, err := s.resourcePool(ctx, tx, pool)
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", err)
|
||||
}
|
||||
for _, member := range members {
|
||||
if !member.Enabled || member.Quarantined {
|
||||
continue
|
||||
}
|
||||
var count int
|
||||
if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if count != 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, member.Member, visit.RunID, visit.Stage, visit.Sequence, lease.token, pool); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
lease.Resource = member.Member
|
||||
lease.pool = pool
|
||||
lease.Target = poolTarget(member)
|
||||
if err := tx.Commit(); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
return lease, nil, nil, nil
|
||||
}
|
||||
wait := &protocol.ResourceWait{Resource: pool, Stage: visit.Stage, Sequence: visit.Sequence, QueuedSince: time.Now().UTC().Format(time.RFC3339Nano)}
|
||||
encoded, err := json.Marshal(wait)
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_waiters(resource_key,pool_name,run_id,stage_key,sequence,token,queued_at) VALUES(?,?,?,?,?,?,?)`, pool, pool, visit.RunID, visit.Stage, visit.Sequence, lease.token, wait.QueuedSince); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='queued',current_stage=?,resource_wait=? WHERE id=? AND status IN ('queued','running')", visit.Stage, string(encoded), visit.RunID)); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
if s.resourceWaiters == nil {
|
||||
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
|
||||
}
|
||||
ch := make(chan ResourceLease, 1)
|
||||
s.resourceWaiters[visit] = ch
|
||||
return ResourceLease{}, ch, wait, nil
|
||||
}
|
||||
|
||||
// RequestResource acquires immediately or registers one FIFO waiter/channel. The
|
||||
@@ -226,8 +408,12 @@ func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (Resou
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("run is not active")
|
||||
}
|
||||
var existing ResourceLease
|
||||
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token)
|
||||
var existingPool string
|
||||
err = tx.QueryRowContext(ctx, "SELECT resource_key,run_id,stage_key,sequence,token,pool_name FROM resource_owners WHERE run_id=?", visit.RunID).Scan(&existing.Resource, &existing.RunID, &existing.Stage, &existing.Sequence, &existing.token, &existingPool)
|
||||
if err == nil {
|
||||
if existingPool != "" {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns a pooled resource visit")
|
||||
}
|
||||
if existing.ResourceVisit != visit {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("run already owns another resource visit")
|
||||
}
|
||||
@@ -236,8 +422,8 @@ func (s *Store) RequestResource(ctx context.Context, visit ResourceVisit) (Resou
|
||||
if err != sql.ErrNoRows {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token)
|
||||
VALUES(?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token)
|
||||
result, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name)
|
||||
VALUES(?,?,?,?,?,?) ON CONFLICT(resource_key) DO NOTHING`, visit.Resource, visit.RunID, visit.Stage, visit.Sequence, lease.token, "")
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
@@ -349,11 +535,18 @@ func (s *Store) ConfirmRecoveredSession(ctx context.Context, owner SessionOwner)
|
||||
// SetResourceCleanup records positive quiescence durably before release. False
|
||||
// quarantines the owner; neither age nor another run can remove the lease.
|
||||
func (s *Store) SetResourceCleanup(ctx context.Context, lease ResourceLease, confirmed bool) error {
|
||||
if lease.pool != "" && (lease.Target == nil || !ValidResourceKey(lease.Target.Member)) {
|
||||
return fmt.Errorf("invalid pooled resource lease")
|
||||
}
|
||||
diagnostic := ""
|
||||
if !confirmed {
|
||||
diagnostic = "Cleanup/restoration unconfirmed; resource quarantined pending inspected reconciliation"
|
||||
}
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, lease.Resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
|
||||
resource := lease.Resource
|
||||
if lease.pool != "" {
|
||||
resource = lease.Target.Member
|
||||
}
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE resource_owners SET cleanup_confirmed=?,diagnostic=? WHERE resource_key=? AND run_id=? AND stage_key=? AND sequence=? AND token=?`, confirmed, diagnostic, resource, lease.RunID, lease.Stage, lease.Sequence, lease.token))
|
||||
}
|
||||
|
||||
// ReleaseResource requires the complete capability AND positive cleanup evidence.
|
||||
@@ -370,6 +563,14 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
ownerResource := lease.Resource
|
||||
if lease.pool != "" {
|
||||
if lease.Target == nil || !ValidResourceKey(lease.Target.Member) {
|
||||
return fmt.Errorf("invalid pooled resource lease")
|
||||
}
|
||||
ownerResource = lease.Target.Member
|
||||
}
|
||||
lease.Resource = ownerResource
|
||||
n, err := deleteResourceOwner(ctx, tx, lease)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -377,7 +578,7 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
next, err := s.handoffResourceTx(ctx, tx, lease.Resource)
|
||||
next, err := s.handoffResourceTx(ctx, tx, ownerResource)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -396,25 +597,86 @@ func (s *Store) releaseResourceLocked(ctx context.Context, lease ResourceLease)
|
||||
func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource string) (ResourceLease, error) {
|
||||
var next ResourceLease
|
||||
next.Resource = resource
|
||||
err := tx.QueryRowContext(ctx, `SELECT run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? ORDER BY queue_order LIMIT 1`, resource).Scan(&next.RunID, &next.Stage, &next.Sequence, &next.token)
|
||||
if err == sql.ErrNoRows {
|
||||
return ResourceLease{}, nil
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT queue_order,resource_key,pool_name,run_id,stage_key,sequence,token FROM resource_waiters WHERE resource_key=? OR pool_name<>'' ORDER BY queue_order`, resource)
|
||||
if err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if s.resourceWaiters[next.ResourceVisit] == nil {
|
||||
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
|
||||
defer rows.Close()
|
||||
var selected *protocol.ResourceTarget
|
||||
var queue int64
|
||||
var waiterResource string
|
||||
for rows.Next() {
|
||||
var candidate ResourceLease
|
||||
var candidatePool string
|
||||
if err := rows.Scan(&queue, &waiterResource, &candidatePool, &candidate.RunID, &candidate.Stage, &candidate.Sequence, &candidate.token); err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if candidatePool != "" {
|
||||
candidate.Resource = candidatePool
|
||||
candidate.pool = candidatePool
|
||||
} else {
|
||||
candidate.Resource = resource
|
||||
}
|
||||
if candidatePool == "" {
|
||||
if waiterResource != resource {
|
||||
continue
|
||||
}
|
||||
selected = &protocol.ResourceTarget{Member: resource}
|
||||
} else {
|
||||
members, e := s.resourcePool(ctx, tx, candidatePool)
|
||||
if e != nil {
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, member := range members {
|
||||
if member.Member != resource || !member.Enabled || member.Quarantined {
|
||||
continue
|
||||
}
|
||||
var count int
|
||||
if e = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM resource_owners WHERE resource_key=?", member.Member).Scan(&count); e != nil {
|
||||
return ResourceLease{}, e
|
||||
}
|
||||
if count == 0 {
|
||||
selected, found = poolTarget(member), true
|
||||
}
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if s.resourceWaiters[candidate.ResourceVisit] == nil {
|
||||
if candidatePool == "" {
|
||||
return ResourceLease{}, fmt.Errorf("resource waiter has no live runner; refusing handoff")
|
||||
}
|
||||
continue
|
||||
}
|
||||
next = candidate
|
||||
next.Target = selected
|
||||
break
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token) VALUES(?,?,?,?,?)`, next.Resource, next.RunID, next.Stage, next.Sequence, next.token); err != nil {
|
||||
if err := rows.Err(); err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE run_id=?`, next.RunID); err != nil {
|
||||
if next.token == "" {
|
||||
return ResourceLease{}, nil
|
||||
}
|
||||
ownerResource := next.Resource
|
||||
if next.pool != "" {
|
||||
ownerResource = next.Target.Member
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO resource_owners(resource_key,run_id,stage_key,sequence,token,pool_name) VALUES(?,?,?,?,?,?)`, ownerResource, next.RunID, next.Stage, next.Sequence, next.token, next.pool); err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM resource_waiters WHERE queue_order=?`, queue); err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET resource_wait='' WHERE id=?", next.RunID); err != nil {
|
||||
return ResourceLease{}, err
|
||||
}
|
||||
if next.pool == "" {
|
||||
next.Resource = ownerResource
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -39,6 +39,206 @@ func TestResourceDeclaration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolAllocatesConcreteMembersAndExcludesDirectOwner(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "android-targets", Members: []protocol.ResourcePoolMember{
|
||||
{Member: "phone", Enabled: true, Metadata: map[string]string{"kind": "phone"}},
|
||||
{Member: "emulator", Enabled: true, Metadata: map[string]string{"kind": "emulator"}},
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
phone := acquireTestResource(t, s, create("phone"))
|
||||
visit := create("android-targets")
|
||||
lease, ch, _, err := s.RequestResourcePool(ctx, "android-targets", visit)
|
||||
if err != nil || ch != nil || lease.Target == nil || lease.Target.Member != "emulator" {
|
||||
t.Fatalf("pool selected %#v, channel=%v, err=%v", lease.Target, ch, err)
|
||||
}
|
||||
if lease.Target.Metadata["kind"] != "emulator" {
|
||||
t.Fatal("missing bounded target metadata")
|
||||
}
|
||||
releaseTestResource(t, s, lease)
|
||||
releaseTestResource(t, s, phone)
|
||||
}
|
||||
|
||||
func TestResourcePoolFIFOWakeAndQuarantineIsolation(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
|
||||
{Member: "held", Enabled: true}, {Member: "free", Enabled: true}, {Member: "bad", Enabled: false, Quarantined: true},
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := acquireTestResource(t, s, create("held"))
|
||||
first := create("targets")
|
||||
firstLease, firstCh, _, err := s.RequestResourcePool(ctx, "targets", first)
|
||||
if err != nil || firstCh != nil || firstLease.Target == nil {
|
||||
t.Fatalf("expected immediate free member: %v", err)
|
||||
}
|
||||
// The second request waits once both concrete members are held.
|
||||
second := create("targets")
|
||||
_, secondCh, _, err := s.RequestResourcePool(ctx, "targets", second)
|
||||
if err != nil || secondCh == nil {
|
||||
t.Fatalf("expected pool waiter: %v", err)
|
||||
}
|
||||
releaseTestResource(t, s, firstLease)
|
||||
select {
|
||||
case next := <-secondCh:
|
||||
if next.Target == nil || next.Target.Member != "free" {
|
||||
t.Fatalf("wrong handoff: %#v", next.Target)
|
||||
}
|
||||
s.AcceptResource(next)
|
||||
releaseTestResource(t, s, next)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pool waiter not woken")
|
||||
}
|
||||
releaseTestResource(t, s, held)
|
||||
}
|
||||
|
||||
func TestResourcePoolRejectsStaleVisitForExistingOwner(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
|
||||
{Member: "phone", Enabled: true},
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owner := acquireTestResource(t, s, create("phone"))
|
||||
stale := owner.ResourceVisit
|
||||
stale.Resource = "targets"
|
||||
stale.Sequence++
|
||||
if _, ch, _, err := s.RequestResourcePool(ctx, "targets", stale); err == nil || ch != nil {
|
||||
t.Fatalf("stale pool visit was accepted: channel=%v err=%v", ch, err)
|
||||
}
|
||||
if err := s.SetResourceCleanup(ctx, owner, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ReleaseResource(ctx, owner); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolAndDirectVisitsCannotBeReinterpreted(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pooled := create("targets")
|
||||
lease, ch, _, err := s.RequestResourcePool(ctx, "targets", pooled)
|
||||
if err != nil || ch != nil {
|
||||
t.Fatalf("pool acquire: %v %v", ch, err)
|
||||
}
|
||||
if _, ch, _, err := s.RequestResource(ctx, ResourceVisit{Resource: "phone", RunID: pooled.RunID, Stage: pooled.Stage, Sequence: pooled.Sequence}); err == nil || ch != nil {
|
||||
t.Fatalf("pooled owner was accepted as direct lease: channel=%v err=%v", ch, err)
|
||||
}
|
||||
releaseTestResource(t, s, lease)
|
||||
}
|
||||
|
||||
func TestResourcePoolConfigurationAndOwnerSurviveReopen(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
|
||||
{Member: "phone", Enabled: true, Metadata: map[string]string{"kind": "phone"}},
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visit := create("targets")
|
||||
owner, ch, _, err := s.RequestResourcePool(ctx, "targets", visit)
|
||||
if err != nil || ch != nil {
|
||||
t.Fatalf("pool acquire: %v %v", ch, err)
|
||||
}
|
||||
path := s.path
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
recovered, ch, _, err := reopened.RequestResourcePool(ctx, "targets", visit)
|
||||
if err != nil || ch != nil || recovered.Target == nil || recovered.Target.Member != "phone" || recovered.Target.Metadata["kind"] != "phone" {
|
||||
t.Fatalf("reopened pool owner not recovered: target=%#v channel=%v err=%v", recovered.Target, ch, err)
|
||||
}
|
||||
if err := reopened.SetResourceCleanup(ctx, recovered, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := reopened.ReleaseResource(ctx, recovered); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = owner
|
||||
}
|
||||
|
||||
func TestResourcePoolAllocatesDistinctMembersConcurrently(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
|
||||
{Member: "phone", Enabled: true}, {Member: "emulator", Enabled: true},
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visits := []ResourceVisit{create("targets"), create("targets")}
|
||||
leases := make(chan ResourceLease, len(visits))
|
||||
errs := make(chan error, len(visits))
|
||||
var wg sync.WaitGroup
|
||||
for _, visit := range visits {
|
||||
wg.Add(1)
|
||||
go func(v ResourceVisit) {
|
||||
defer wg.Done()
|
||||
lease, ch, _, err := s.RequestResourcePool(ctx, "targets", v)
|
||||
if err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
if ch != nil {
|
||||
select {
|
||||
case lease = <-ch:
|
||||
case <-time.After(time.Second):
|
||||
errs <- fmt.Errorf("pool request remained queued")
|
||||
return
|
||||
}
|
||||
s.AcceptResource(lease)
|
||||
}
|
||||
leases <- lease
|
||||
}(visit)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for range visits {
|
||||
lease := <-leases
|
||||
if lease.Target == nil || got[lease.Target.Member] {
|
||||
t.Fatalf("members were not distinct: %#v", lease.Target)
|
||||
}
|
||||
got[lease.Target.Member] = true
|
||||
releaseTestResource(t, s, lease)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolRejectsInvalidDeclarations(t *testing.T) {
|
||||
s, _ := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
cases := []protocol.ResourcePool{
|
||||
{Name: "missing-members"},
|
||||
{Name: "duplicate", Members: []protocol.ResourcePoolMember{{Member: "phone"}, {Member: "phone"}}},
|
||||
{Name: "bad-member", Members: []protocol.ResourcePoolMember{{Member: "not valid"}}},
|
||||
{Name: "bad-metadata", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{"bad key": "x"}}}},
|
||||
}
|
||||
for _, pool := range cases {
|
||||
if err := s.ConfigureResourcePool(ctx, pool); err == nil {
|
||||
t.Fatalf("accepted invalid pool %#v", pool)
|
||||
}
|
||||
}
|
||||
visit := ResourceVisit{Resource: "unknown", RunID: 1, Stage: "work", Sequence: 1}
|
||||
if _, _, _, err := s.RequestResourcePool(ctx, "unknown", visit); err == nil {
|
||||
t.Fatal("accepted unknown pool")
|
||||
}
|
||||
}
|
||||
|
||||
func resourceStore(t *testing.T) (*Store, func(string) ResourceVisit) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -58,7 +58,7 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints (
|
||||
source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
|
||||
CREATE TABLE IF NOT EXISTS resource_owners (
|
||||
resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
|
||||
stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL,
|
||||
stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, pool_name TEXT NOT NULL DEFAULT '',
|
||||
cleanup_confirmed INTEGER NOT NULL DEFAULT 0, diagnostic TEXT NOT NULL DEFAULT '',
|
||||
acquired_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
|
||||
CREATE INDEX IF NOT EXISTS resource_owner_run ON resource_owners(run_id);
|
||||
@@ -71,6 +71,17 @@ CREATE INDEX IF NOT EXISTS resource_wait_fifo ON resource_waiters(resource_key,q
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS resource_pools (
|
||||
name TEXT PRIMARY KEY, members_json TEXT NOT NULL, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.addColumnIfMissing("resource_waiters", "pool_name", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.addColumnIfMissing("resource_owners", "pool_name", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.migrateRunTitles(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,6 +6,25 @@ type ResourceInspection struct {
|
||||
Waiters []ResourceWaiter `json:"waiters,omitempty"`
|
||||
Audits []ResourceRecoveryAudit `json:"audits,omitempty"`
|
||||
}
|
||||
|
||||
// ResourceTarget is bounded, non-secret metadata for the concrete member that
|
||||
// was selected. Credentials and capabilities are intentionally absent.
|
||||
type ResourceTarget struct {
|
||||
Member string `json:"member"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ResourcePoolMember struct {
|
||||
Member string `json:"member"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Quarantined bool `json:"quarantined,omitempty"`
|
||||
}
|
||||
|
||||
type ResourcePool struct {
|
||||
Name string `json:"name"`
|
||||
Members []ResourcePoolMember `json:"members"`
|
||||
}
|
||||
type ResourceOwner struct {
|
||||
RunID int64 `json:"run_id"`
|
||||
Stage string `json:"stage"`
|
||||
|
||||
Reference in New Issue
Block a user