Wire pool resource targets through runner and repair quarantined-member handoff

This commit is contained in:
2026-10-18 23:20:55 +01:00
parent a4c7149060
commit a022b58c5c
10 changed files with 246 additions and 15 deletions
@@ -0,0 +1,11 @@
# Resource-pool runner handoff — 2026-10-18
Stage `resources` declarations now accept exactly one of `exclusive` or `pool`.
Pool stages wait on the named pool, then receive the selected concrete member and
bounded metadata through explicit runtime context. The model input remains
unchanged and capability tokens remain private to the store. Retained control
evidence records the concrete target; queued run state names the pool while an
active owner names the selected member.
This is parser/runner handoff only. It does not add Android definitions, pool
configuration rollout, device setup, probing, or deployment.
+7 -2
View File
@@ -45,8 +45,8 @@ func decodeControl(text string, value any) error {
func parseControl(s *Stage, fields map[string]string) error {
if text, ok := fields["resources"]; ok {
if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !ValidResourceKey(s.Resources.Exclusive) {
return fmt.Errorf("resources must contain exactly one exclusive key (1–128 ASCII letters, digits, _ or -; start with a letter)")
if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !validStageResources(*s.Resources) {
return fmt.Errorf("resources must contain exactly one exclusive or pool key (1–128 ASCII letters, digits, _ or -; start with a letter)")
}
delete(fields, "resources")
}
@@ -81,6 +81,11 @@ func parseControl(s *Stage, fields map[string]string) error {
return nil
}
func validStageResources(resources protocol.StageResources) bool {
return (ValidResourceKey(resources.Exclusive) && resources.Pool == "") ||
(resources.Exclusive == "" && ValidResourceKey(resources.Pool))
}
func validateControl(s Stage, stages map[string]bool) error {
if s.Resources != nil && s.Type == "human" {
return fmt.Errorf("stage %s: human stages cannot declare resources", s.Key)
+27 -3
View File
@@ -200,6 +200,7 @@ type ResourceLease struct {
const maxPoolMembers = 32
const maxTargetMetadata = 16
const maxTargetMetadataKey = 128
const maxTargetValue = 128
func validateTargetMetadata(metadata map[string]string) error {
@@ -207,7 +208,7 @@ func validateTargetMetadata(metadata map[string]string) error {
return fmt.Errorf("too many target metadata fields")
}
for k, v := range metadata {
if !identifier.MatchString(k) || len(v) > maxTargetValue || !utf8.ValidString(v) {
if !identifier.MatchString(k) || len(k) > maxTargetMetadataKey || len(v) > maxTargetValue || !utf8.ValidString(v) {
return fmt.Errorf("invalid target metadata")
}
}
@@ -330,6 +331,29 @@ func (s *Store) RequestResourcePool(ctx context.Context, pool string, visit Reso
if err != sql.ErrNoRows {
return ResourceLease{}, nil, nil, err
}
// A queued visit may be re-entered after the process that owned its live
// channel was restarted. Reattach to the durable waiter instead of trying
// to insert a second reservation.
var persistedPool string
err = tx.QueryRowContext(ctx, "SELECT pool_name FROM resource_waiters WHERE run_id=? AND stage_key=? AND sequence=?", visit.RunID, visit.Stage, visit.Sequence).Scan(&persistedPool)
if err == nil {
if persistedPool != pool {
return ResourceLease{}, nil, nil, fmt.Errorf("resource waiter pool visit is stale")
}
wait, waitErr := s.resourceWait(ctx, visit.RunID)
if waitErr != nil {
return ResourceLease{}, nil, nil, waitErr
}
if s.resourceWaiters == nil {
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
}
ch := make(chan ResourceLease, 1)
s.resourceWaiters[visit] = ch
return ResourceLease{}, ch, wait, nil
}
if err != sql.ErrNoRows {
return ResourceLease{}, nil, nil, err
}
members, err := s.resourcePool(ctx, tx, pool)
if err != nil {
return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", err)
@@ -629,7 +653,7 @@ func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource stri
}
found := false
for _, member := range members {
if member.Member != resource || !member.Enabled || member.Quarantined {
if !member.Enabled || member.Quarantined {
continue
}
var count int
@@ -638,8 +662,8 @@ func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource stri
}
if count == 0 {
selected, found = poolTarget(member), true
break
}
break
}
if !found {
continue
+49
View File
@@ -39,6 +39,19 @@ func TestResourceDeclaration(t *testing.T) {
}
}
func TestResourcePoolDeclaration(t *testing.T) {
base := Template([]string{"work"}, true)
graph, err := Parse(base + "```resources\n{\"pool\":\"android-targets\"}\n```\n")
if err != nil || graph.Stages[0].Resources == nil || graph.Stages[0].Resources.Pool != "android-targets" {
t.Fatalf("pool declaration: %#v, %v", graph.Stages[0].Resources, err)
}
for _, value := range []string{`{"exclusive":"phone","pool":"targets"}`, `{"pool":""}`, `{"pool":"a b"}`} {
if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil {
t.Fatalf("accepted invalid pool declaration %s", value)
}
}
}
func TestResourcePoolAllocatesConcreteMembersAndExcludesDirectOwner(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
@@ -95,6 +108,41 @@ func TestResourcePoolFIFOWakeAndQuarantineIsolation(t *testing.T) {
releaseTestResource(t, s, held)
}
func TestResourcePoolHandoffUsesAnotherMemberWhenReleasedTargetQuarantined(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
pool := protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
{Member: "phone", Enabled: true}, {Member: "emulator", Enabled: false},
}}
if err := s.ConfigureResourcePool(ctx, pool); err != nil {
t.Fatal(err)
}
owner, ready, _, err := s.RequestResourcePool(ctx, pool.Name, create(pool.Name))
if err != nil || ready != nil || owner.Target.Member != "phone" {
t.Fatalf("owner: %+v %v", owner, err)
}
_, waiter, _, err := s.RequestResourcePool(ctx, pool.Name, create(pool.Name))
if err != nil || waiter == nil {
t.Fatalf("waiter: %v", err)
}
pool.Members[0].Quarantined = true
pool.Members[1].Enabled = true
if err := s.ConfigureResourcePool(ctx, pool); err != nil {
t.Fatal(err)
}
releaseTestResource(t, s, owner)
select {
case next := <-waiter:
if next.Target == nil || next.Target.Member != "emulator" {
t.Fatalf("wrong target: %+v", next.Target)
}
s.AcceptResource(next)
releaseTestResource(t, s, next)
case <-time.After(time.Second):
t.Fatal("quarantined released phone blocked the free emulator")
}
}
func TestResourcePoolRejectsStaleVisitForExistingOwner(t *testing.T) {
s, create := resourceStore(t)
ctx := context.Background()
@@ -227,6 +275,7 @@ func TestResourcePoolRejectsInvalidDeclarations(t *testing.T) {
{Name: "duplicate", Members: []protocol.ResourcePoolMember{{Member: "phone"}, {Member: "phone"}}},
{Name: "bad-member", Members: []protocol.ResourcePoolMember{{Member: "not valid"}}},
{Name: "bad-metadata", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{"bad key": "x"}}}},
{Name: "oversized-metadata-key", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{strings.Repeat("k", maxTargetMetadataKey+1): "x"}}}},
}
for _, pool := range cases {
if err := s.ConfigureResourcePool(ctx, pool); err == nil {
+1
View File
@@ -41,6 +41,7 @@ type Stage struct {
// StageResources declares one capacity-one resource, local to this store authority.
type StageResources struct {
Exclusive string `json:"exclusive"`
Pool string `json:"pool"`
}
// Limits are enforced by the runner and never included in the model prompt.
+6 -5
View File
@@ -331,11 +331,12 @@ type MemoryRead struct {
}
type ControlEvidence struct {
Visit int `json:"visit"`
MaxVisits int `json:"max_visits"`
LimitReached bool `json:"limit_reached,omitempty"`
ModelTarget string `json:"model_target,omitempty"`
Memory map[string]MemoryRead `json:"memory,omitempty"`
Visit int `json:"visit"`
MaxVisits int `json:"max_visits"`
LimitReached bool `json:"limit_reached,omitempty"`
ModelTarget string `json:"model_target,omitempty"`
Memory map[string]MemoryRead `json:"memory,omitempty"`
ResourceTarget *ResourceTarget `json:"resource_target,omitempty"`
}
// StageFileChanges is observed workspace evidence, not a model's claim of authorship.
+13 -1
View File
@@ -11,7 +11,19 @@ import (
)
func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) {
lease, ready, wait, err := r.store.RequestResource(ctx, visit)
return r.acquireResourceWithPool(ctx, visit, "", workspace, completed)
}
func (r *runner) acquireResourceWithPool(ctx context.Context, visit pipeline.ResourceVisit, pool string, workspace string, completed int) (pipeline.ResourceLease, error) {
var lease pipeline.ResourceLease
var ready <-chan pipeline.ResourceLease
var wait *protocol.ResourceWait
var err error
if pool != "" {
lease, ready, wait, err = r.store.RequestResourcePool(ctx, pool, visit)
} else {
lease, ready, wait, err = r.store.RequestResource(ctx, visit)
}
if err != nil {
return lease, err
}
+91
View File
@@ -137,6 +137,97 @@ func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) {
}
}
func TestResourcePoolRunnerHandoffUsesConcreteTargetAfterQueue(t *testing.T) {
r, events, _ := resourceRunner(t)
ctx := context.Background()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"pool\":\"targets\"}\n```\n"
if _, err := r.store.Submit(ctx, "pool-runner", source); err != nil {
t.Fatal(err)
}
items, err := r.store.List(ctx)
if err != nil || len(items) != 1 {
t.Fatal("pool definition missing", err)
}
definition, err := r.store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
if err := r.store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "phone-serial", "type": "phone"}},
{Member: "emulator", Enabled: true, Metadata: map[string]string{"serial": "emu-serial", "type": "emulator"}},
}}); err != nil {
t.Fatal(err)
}
ownerRun, _, err := r.store.CreateRun(ctx, definition, "phone-owner", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
phone, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "phone", RunID: ownerRun, Stage: "work", Sequence: 1})
if err != nil || ch != nil {
t.Fatalf("phone owner acquire: channel=%v err=%v", ch, err)
}
defer r.releaseResource(phone, true)
aStarted, releaseA := make(chan struct{}), make(chan struct{})
bRan := make(chan *protocol.ResourceTarget, 1)
r.execute = func(stageCtx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
target, ok := stage.ResourceTarget(stageCtx)
if !ok || target == nil {
return stage.Result{}, errors.New("missing resource target context")
}
if input["prompt"] == "A" {
if target.Member != "emulator" || target.Metadata["serial"] != "emu-serial" || target.Metadata["type"] != "emulator" {
return stage.Result{}, fmt.Errorf("wrong first target: %#v", target)
}
close(aStarted)
<-releaseA
} else {
if deadline, ok := stageCtx.Deadline(); !ok || time.Until(deadline) < 15*time.Millisecond {
return stage.Result{}, errors.New("execution deadline was consumed in pool queue")
}
bRan <- target
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
start := func(key, prompt string) *protocol.RunDetail {
run, err := r.start(protocol.Message{PipelineID: definition.ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
if err != nil {
t.Fatal(err)
}
return run
}
a := start("pool-a", "A")
<-aStarted
b := start("pool-b", "B")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
time.Sleep(80 * time.Millisecond)
select {
case <-bRan:
t.Fatal("pool waiter ran before either member was released")
default:
}
if err := r.releaseResource(phone, true); err != nil {
t.Fatal(err)
}
var target *protocol.ResourceTarget
select {
case target = <-bRan:
case <-time.After(3 * time.Second):
t.Fatal("pool waiter did not resume")
}
if target.Member != "phone" || target.Metadata["serial"] != "phone-serial" {
t.Fatalf("wrong queued target: %#v", target)
}
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
close(releaseA)
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
r.wg.Wait()
detail, err := r.store.GetRun(ctx, b.ID)
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Control.ResourceTarget == nil {
t.Fatalf("pool target evidence missing: detail=%#v err=%v", detail, err)
}
}
func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) {
r, events, start := resourceRunner(t)
started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{})
+15 -4
View File
@@ -1241,11 +1241,16 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
_, declaresSkip := def.Edges["skipped"]
skip := !isHuman && declaresSkip && !state.uiSurface
if selected.Resources != nil && !skip {
if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) {
if isHuman || (selected.Resources.Exclusive == "" && selected.Resources.Pool == "") || (selected.Resources.Exclusive != "" && selected.Resources.Pool != "") || (selected.Resources.Exclusive != "" && !pipeline.ValidResourceKey(selected.Resources.Exclusive)) || (selected.Resources.Pool != "" && !pipeline.ValidResourceKey(selected.Resources.Pool)) {
finish("failed", "error", "invalid persisted resource declaration")
return
}
lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed)
resource := selected.Resources.Exclusive
pool := selected.Resources.Pool
if pool != "" {
resource = pool
}
lease, err := r.acquireResourceWithPool(ctx, pipeline.ResourceVisit{Resource: resource, RunID: id, Stage: key, Sequence: sequence}, pool, workspace, state.completed)
if err != nil {
if ctx.Err() != nil {
finish("canceled", "", err.Error())
@@ -1392,6 +1397,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
var piClient *stage.PiClient
var err error
stageCtx := ctx
if held != nil && held.Target != nil {
stageCtx = stage.WithResourceTarget(stageCtx, held.Target)
}
var modelConfig stage.ModelConfig
var budgetBefore stage.BudgetSnapshot
if stage.EffectiveType(def.Type) == stage.ModelType {
@@ -1408,7 +1416,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
// its cumulative usage before the visit and record only this visit's
// increment on the step, never the cumulative total again.
budgetBefore = budgets[key].Snapshot()
stageCtx = stage.WithBudget(ctx, budgets[key])
stageCtx = stage.WithBudget(stageCtx, budgets[key])
}
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
before := filediff.Capture(snapshotCtx, workspace)
@@ -1470,7 +1478,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
client.Transcript = plain.FormatAudit
} else {
result, err = stage.DefaultRegistry().Run(ctx, def, input)
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
}
}()
if held != nil {
@@ -1571,6 +1579,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
step.Target = def.Edges[step.Outcome]
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
if held != nil && held.Target != nil {
step.Control.ResourceTarget = held.Target
}
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
for _, outcome := range selected.Limits.On {
if outcome == step.Outcome {
+26
View File
@@ -5,8 +5,34 @@ package stage
import (
"context"
"fmt"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type resourceTargetContextKey struct{}
// WithResourceTarget attaches the server-granted concrete target to a stage's
// runtime context. It is deliberately separate from model input and contains
// no capability or credential material.
func WithResourceTarget(ctx context.Context, target *protocol.ResourceTarget) context.Context {
if target == nil {
return ctx
}
copy := &protocol.ResourceTarget{Member: target.Member}
if target.Metadata != nil {
copy.Metadata = map[string]string{}
for key, value := range target.Metadata {
copy.Metadata[key] = value
}
}
return context.WithValue(ctx, resourceTargetContextKey{}, copy)
}
func ResourceTarget(ctx context.Context) (*protocol.ResourceTarget, bool) {
target, ok := ctx.Value(resourceTargetContextKey{}).(*protocol.ResourceTarget)
return target, ok
}
// Values are runtime data, not the prose output contract in a definition.
type Values map[string]any