Wire pool resource targets through runner and repair quarantined-member handoff
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
# Resource-pool runner handoff — 2026-10-18
|
||||
|
||||
Stage `resources` declarations now accept exactly one of `exclusive` or `pool`.
|
||||
Pool stages wait on the named pool, then receive the selected concrete member and
|
||||
bounded metadata through explicit runtime context. The model input remains
|
||||
unchanged and capability tokens remain private to the store. Retained control
|
||||
evidence records the concrete target; queued run state names the pool while an
|
||||
active owner names the selected member.
|
||||
|
||||
This is parser/runner handoff only. It does not add Android definitions, pool
|
||||
configuration rollout, device setup, probing, or deployment.
|
||||
@@ -45,8 +45,8 @@ func decodeControl(text string, value any) error {
|
||||
|
||||
func parseControl(s *Stage, fields map[string]string) error {
|
||||
if text, ok := fields["resources"]; ok {
|
||||
if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !ValidResourceKey(s.Resources.Exclusive) {
|
||||
return fmt.Errorf("resources must contain exactly one exclusive key (1–128 ASCII letters, digits, _ or -; start with a letter)")
|
||||
if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !validStageResources(*s.Resources) {
|
||||
return fmt.Errorf("resources must contain exactly one exclusive or pool key (1–128 ASCII letters, digits, _ or -; start with a letter)")
|
||||
}
|
||||
delete(fields, "resources")
|
||||
}
|
||||
@@ -81,6 +81,11 @@ func parseControl(s *Stage, fields map[string]string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validStageResources(resources protocol.StageResources) bool {
|
||||
return (ValidResourceKey(resources.Exclusive) && resources.Pool == "") ||
|
||||
(resources.Exclusive == "" && ValidResourceKey(resources.Pool))
|
||||
}
|
||||
|
||||
func validateControl(s Stage, stages map[string]bool) error {
|
||||
if s.Resources != nil && s.Type == "human" {
|
||||
return fmt.Errorf("stage %s: human stages cannot declare resources", s.Key)
|
||||
|
||||
@@ -200,6 +200,7 @@ type ResourceLease struct {
|
||||
|
||||
const maxPoolMembers = 32
|
||||
const maxTargetMetadata = 16
|
||||
const maxTargetMetadataKey = 128
|
||||
const maxTargetValue = 128
|
||||
|
||||
func validateTargetMetadata(metadata map[string]string) error {
|
||||
@@ -207,7 +208,7 @@ func validateTargetMetadata(metadata map[string]string) error {
|
||||
return fmt.Errorf("too many target metadata fields")
|
||||
}
|
||||
for k, v := range metadata {
|
||||
if !identifier.MatchString(k) || len(v) > maxTargetValue || !utf8.ValidString(v) {
|
||||
if !identifier.MatchString(k) || len(k) > maxTargetMetadataKey || len(v) > maxTargetValue || !utf8.ValidString(v) {
|
||||
return fmt.Errorf("invalid target metadata")
|
||||
}
|
||||
}
|
||||
@@ -330,6 +331,29 @@ func (s *Store) RequestResourcePool(ctx context.Context, pool string, visit Reso
|
||||
if err != sql.ErrNoRows {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
// A queued visit may be re-entered after the process that owned its live
|
||||
// channel was restarted. Reattach to the durable waiter instead of trying
|
||||
// to insert a second reservation.
|
||||
var persistedPool string
|
||||
err = tx.QueryRowContext(ctx, "SELECT pool_name FROM resource_waiters WHERE run_id=? AND stage_key=? AND sequence=?", visit.RunID, visit.Stage, visit.Sequence).Scan(&persistedPool)
|
||||
if err == nil {
|
||||
if persistedPool != pool {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("resource waiter pool visit is stale")
|
||||
}
|
||||
wait, waitErr := s.resourceWait(ctx, visit.RunID)
|
||||
if waitErr != nil {
|
||||
return ResourceLease{}, nil, nil, waitErr
|
||||
}
|
||||
if s.resourceWaiters == nil {
|
||||
s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease)
|
||||
}
|
||||
ch := make(chan ResourceLease, 1)
|
||||
s.resourceWaiters[visit] = ch
|
||||
return ResourceLease{}, ch, wait, nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return ResourceLease{}, nil, nil, err
|
||||
}
|
||||
members, err := s.resourcePool(ctx, tx, pool)
|
||||
if err != nil {
|
||||
return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", err)
|
||||
@@ -629,7 +653,7 @@ func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource stri
|
||||
}
|
||||
found := false
|
||||
for _, member := range members {
|
||||
if member.Member != resource || !member.Enabled || member.Quarantined {
|
||||
if !member.Enabled || member.Quarantined {
|
||||
continue
|
||||
}
|
||||
var count int
|
||||
@@ -638,8 +662,8 @@ func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource stri
|
||||
}
|
||||
if count == 0 {
|
||||
selected, found = poolTarget(member), true
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
|
||||
@@ -39,6 +39,19 @@ func TestResourceDeclaration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolDeclaration(t *testing.T) {
|
||||
base := Template([]string{"work"}, true)
|
||||
graph, err := Parse(base + "```resources\n{\"pool\":\"android-targets\"}\n```\n")
|
||||
if err != nil || graph.Stages[0].Resources == nil || graph.Stages[0].Resources.Pool != "android-targets" {
|
||||
t.Fatalf("pool declaration: %#v, %v", graph.Stages[0].Resources, err)
|
||||
}
|
||||
for _, value := range []string{`{"exclusive":"phone","pool":"targets"}`, `{"pool":""}`, `{"pool":"a b"}`} {
|
||||
if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil {
|
||||
t.Fatalf("accepted invalid pool declaration %s", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolAllocatesConcreteMembersAndExcludesDirectOwner(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
@@ -95,6 +108,41 @@ func TestResourcePoolFIFOWakeAndQuarantineIsolation(t *testing.T) {
|
||||
releaseTestResource(t, s, held)
|
||||
}
|
||||
|
||||
func TestResourcePoolHandoffUsesAnotherMemberWhenReleasedTargetQuarantined(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
pool := protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
|
||||
{Member: "phone", Enabled: true}, {Member: "emulator", Enabled: false},
|
||||
}}
|
||||
if err := s.ConfigureResourcePool(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owner, ready, _, err := s.RequestResourcePool(ctx, pool.Name, create(pool.Name))
|
||||
if err != nil || ready != nil || owner.Target.Member != "phone" {
|
||||
t.Fatalf("owner: %+v %v", owner, err)
|
||||
}
|
||||
_, waiter, _, err := s.RequestResourcePool(ctx, pool.Name, create(pool.Name))
|
||||
if err != nil || waiter == nil {
|
||||
t.Fatalf("waiter: %v", err)
|
||||
}
|
||||
pool.Members[0].Quarantined = true
|
||||
pool.Members[1].Enabled = true
|
||||
if err := s.ConfigureResourcePool(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
releaseTestResource(t, s, owner)
|
||||
select {
|
||||
case next := <-waiter:
|
||||
if next.Target == nil || next.Target.Member != "emulator" {
|
||||
t.Fatalf("wrong target: %+v", next.Target)
|
||||
}
|
||||
s.AcceptResource(next)
|
||||
releaseTestResource(t, s, next)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("quarantined released phone blocked the free emulator")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolRejectsStaleVisitForExistingOwner(t *testing.T) {
|
||||
s, create := resourceStore(t)
|
||||
ctx := context.Background()
|
||||
@@ -227,6 +275,7 @@ func TestResourcePoolRejectsInvalidDeclarations(t *testing.T) {
|
||||
{Name: "duplicate", Members: []protocol.ResourcePoolMember{{Member: "phone"}, {Member: "phone"}}},
|
||||
{Name: "bad-member", Members: []protocol.ResourcePoolMember{{Member: "not valid"}}},
|
||||
{Name: "bad-metadata", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{"bad key": "x"}}}},
|
||||
{Name: "oversized-metadata-key", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{strings.Repeat("k", maxTargetMetadataKey+1): "x"}}}},
|
||||
}
|
||||
for _, pool := range cases {
|
||||
if err := s.ConfigureResourcePool(ctx, pool); err == nil {
|
||||
|
||||
@@ -41,6 +41,7 @@ type Stage struct {
|
||||
// StageResources declares one capacity-one resource, local to this store authority.
|
||||
type StageResources struct {
|
||||
Exclusive string `json:"exclusive"`
|
||||
Pool string `json:"pool"`
|
||||
}
|
||||
|
||||
// Limits are enforced by the runner and never included in the model prompt.
|
||||
|
||||
@@ -331,11 +331,12 @@ type MemoryRead struct {
|
||||
}
|
||||
|
||||
type ControlEvidence struct {
|
||||
Visit int `json:"visit"`
|
||||
MaxVisits int `json:"max_visits"`
|
||||
LimitReached bool `json:"limit_reached,omitempty"`
|
||||
ModelTarget string `json:"model_target,omitempty"`
|
||||
Memory map[string]MemoryRead `json:"memory,omitempty"`
|
||||
Visit int `json:"visit"`
|
||||
MaxVisits int `json:"max_visits"`
|
||||
LimitReached bool `json:"limit_reached,omitempty"`
|
||||
ModelTarget string `json:"model_target,omitempty"`
|
||||
Memory map[string]MemoryRead `json:"memory,omitempty"`
|
||||
ResourceTarget *ResourceTarget `json:"resource_target,omitempty"`
|
||||
}
|
||||
|
||||
// StageFileChanges is observed workspace evidence, not a model's claim of authorship.
|
||||
|
||||
@@ -11,7 +11,19 @@ import (
|
||||
)
|
||||
|
||||
func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) {
|
||||
lease, ready, wait, err := r.store.RequestResource(ctx, visit)
|
||||
return r.acquireResourceWithPool(ctx, visit, "", workspace, completed)
|
||||
}
|
||||
|
||||
func (r *runner) acquireResourceWithPool(ctx context.Context, visit pipeline.ResourceVisit, pool string, workspace string, completed int) (pipeline.ResourceLease, error) {
|
||||
var lease pipeline.ResourceLease
|
||||
var ready <-chan pipeline.ResourceLease
|
||||
var wait *protocol.ResourceWait
|
||||
var err error
|
||||
if pool != "" {
|
||||
lease, ready, wait, err = r.store.RequestResourcePool(ctx, pool, visit)
|
||||
} else {
|
||||
lease, ready, wait, err = r.store.RequestResource(ctx, visit)
|
||||
}
|
||||
if err != nil {
|
||||
return lease, err
|
||||
}
|
||||
|
||||
@@ -137,6 +137,97 @@ func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolRunnerHandoffUsesConcreteTargetAfterQueue(t *testing.T) {
|
||||
r, events, _ := resourceRunner(t)
|
||||
ctx := context.Background()
|
||||
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"pool\":\"targets\"}\n```\n"
|
||||
if _, err := r.store.Submit(ctx, "pool-runner", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := r.store.List(ctx)
|
||||
if err != nil || len(items) != 1 {
|
||||
t.Fatal("pool definition missing", err)
|
||||
}
|
||||
definition, err := r.store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
|
||||
{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "phone-serial", "type": "phone"}},
|
||||
{Member: "emulator", Enabled: true, Metadata: map[string]string{"serial": "emu-serial", "type": "emulator"}},
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ownerRun, _, err := r.store.CreateRun(ctx, definition, "phone-owner", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
phone, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "phone", RunID: ownerRun, Stage: "work", Sequence: 1})
|
||||
if err != nil || ch != nil {
|
||||
t.Fatalf("phone owner acquire: channel=%v err=%v", ch, err)
|
||||
}
|
||||
defer r.releaseResource(phone, true)
|
||||
|
||||
aStarted, releaseA := make(chan struct{}), make(chan struct{})
|
||||
bRan := make(chan *protocol.ResourceTarget, 1)
|
||||
r.execute = func(stageCtx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
target, ok := stage.ResourceTarget(stageCtx)
|
||||
if !ok || target == nil {
|
||||
return stage.Result{}, errors.New("missing resource target context")
|
||||
}
|
||||
if input["prompt"] == "A" {
|
||||
if target.Member != "emulator" || target.Metadata["serial"] != "emu-serial" || target.Metadata["type"] != "emulator" {
|
||||
return stage.Result{}, fmt.Errorf("wrong first target: %#v", target)
|
||||
}
|
||||
close(aStarted)
|
||||
<-releaseA
|
||||
} else {
|
||||
if deadline, ok := stageCtx.Deadline(); !ok || time.Until(deadline) < 15*time.Millisecond {
|
||||
return stage.Result{}, errors.New("execution deadline was consumed in pool queue")
|
||||
}
|
||||
bRan <- target
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
start := func(key, prompt string) *protocol.RunDetail {
|
||||
run, err := r.start(protocol.Message{PipelineID: definition.ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return run
|
||||
}
|
||||
a := start("pool-a", "A")
|
||||
<-aStarted
|
||||
b := start("pool-b", "B")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
select {
|
||||
case <-bRan:
|
||||
t.Fatal("pool waiter ran before either member was released")
|
||||
default:
|
||||
}
|
||||
if err := r.releaseResource(phone, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var target *protocol.ResourceTarget
|
||||
select {
|
||||
case target = <-bRan:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("pool waiter did not resume")
|
||||
}
|
||||
if target.Member != "phone" || target.Metadata["serial"] != "phone-serial" {
|
||||
t.Fatalf("wrong queued target: %#v", target)
|
||||
}
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
close(releaseA)
|
||||
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
r.wg.Wait()
|
||||
detail, err := r.store.GetRun(ctx, b.ID)
|
||||
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Control.ResourceTarget == nil {
|
||||
t.Fatalf("pool target evidence missing: detail=%#v err=%v", detail, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
|
||||
@@ -1241,11 +1241,16 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
_, declaresSkip := def.Edges["skipped"]
|
||||
skip := !isHuman && declaresSkip && !state.uiSurface
|
||||
if selected.Resources != nil && !skip {
|
||||
if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) {
|
||||
if isHuman || (selected.Resources.Exclusive == "" && selected.Resources.Pool == "") || (selected.Resources.Exclusive != "" && selected.Resources.Pool != "") || (selected.Resources.Exclusive != "" && !pipeline.ValidResourceKey(selected.Resources.Exclusive)) || (selected.Resources.Pool != "" && !pipeline.ValidResourceKey(selected.Resources.Pool)) {
|
||||
finish("failed", "error", "invalid persisted resource declaration")
|
||||
return
|
||||
}
|
||||
lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed)
|
||||
resource := selected.Resources.Exclusive
|
||||
pool := selected.Resources.Pool
|
||||
if pool != "" {
|
||||
resource = pool
|
||||
}
|
||||
lease, err := r.acquireResourceWithPool(ctx, pipeline.ResourceVisit{Resource: resource, RunID: id, Stage: key, Sequence: sequence}, pool, workspace, state.completed)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", err.Error())
|
||||
@@ -1392,6 +1397,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
var piClient *stage.PiClient
|
||||
var err error
|
||||
stageCtx := ctx
|
||||
if held != nil && held.Target != nil {
|
||||
stageCtx = stage.WithResourceTarget(stageCtx, held.Target)
|
||||
}
|
||||
var modelConfig stage.ModelConfig
|
||||
var budgetBefore stage.BudgetSnapshot
|
||||
if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||
@@ -1408,7 +1416,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
// its cumulative usage before the visit and record only this visit's
|
||||
// increment on the step, never the cumulative total again.
|
||||
budgetBefore = budgets[key].Snapshot()
|
||||
stageCtx = stage.WithBudget(ctx, budgets[key])
|
||||
stageCtx = stage.WithBudget(stageCtx, budgets[key])
|
||||
}
|
||||
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
before := filediff.Capture(snapshotCtx, workspace)
|
||||
@@ -1470,7 +1478,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
|
||||
client.Transcript = plain.FormatAudit
|
||||
} else {
|
||||
result, err = stage.DefaultRegistry().Run(ctx, def, input)
|
||||
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
|
||||
}
|
||||
}()
|
||||
if held != nil {
|
||||
@@ -1571,6 +1579,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
step.Target = def.Edges[step.Outcome]
|
||||
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
|
||||
if held != nil && held.Target != nil {
|
||||
step.Control.ResourceTarget = held.Target
|
||||
}
|
||||
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
|
||||
for _, outcome := range selected.Limits.On {
|
||||
if outcome == step.Outcome {
|
||||
|
||||
@@ -5,8 +5,34 @@ package stage
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
type resourceTargetContextKey struct{}
|
||||
|
||||
// WithResourceTarget attaches the server-granted concrete target to a stage's
|
||||
// runtime context. It is deliberately separate from model input and contains
|
||||
// no capability or credential material.
|
||||
func WithResourceTarget(ctx context.Context, target *protocol.ResourceTarget) context.Context {
|
||||
if target == nil {
|
||||
return ctx
|
||||
}
|
||||
copy := &protocol.ResourceTarget{Member: target.Member}
|
||||
if target.Metadata != nil {
|
||||
copy.Metadata = map[string]string{}
|
||||
for key, value := range target.Metadata {
|
||||
copy.Metadata[key] = value
|
||||
}
|
||||
}
|
||||
return context.WithValue(ctx, resourceTargetContextKey{}, copy)
|
||||
}
|
||||
|
||||
func ResourceTarget(ctx context.Context) (*protocol.ResourceTarget, bool) {
|
||||
target, ok := ctx.Value(resourceTargetContextKey{}).(*protocol.ResourceTarget)
|
||||
return target, ok
|
||||
}
|
||||
|
||||
// Values are runtime data, not the prose output contract in a definition.
|
||||
type Values map[string]any
|
||||
|
||||
|
||||
Reference in New Issue
Block a user