diff --git a/docs/notes/resource-pool-runner-handoff.md b/docs/notes/resource-pool-runner-handoff.md new file mode 100644 index 00000000..d7c9d091 --- /dev/null +++ b/docs/notes/resource-pool-runner-handoff.md @@ -0,0 +1,11 @@ +# Resource-pool runner handoff — 2026-10-18 + +Stage `resources` declarations now accept exactly one of `exclusive` or `pool`. +Pool stages wait on the named pool, then receive the selected concrete member and +bounded metadata through explicit runtime context. The model input remains +unchanged and capability tokens remain private to the store. Retained control +evidence records the concrete target; queued run state names the pool while an +active owner names the selected member. + +This is parser/runner handoff only. It does not add Android definitions, pool +configuration rollout, device setup, probing, or deployment. diff --git a/internal/pipeline/control.go b/internal/pipeline/control.go index 9d32e974..4743bad7 100644 --- a/internal/pipeline/control.go +++ b/internal/pipeline/control.go @@ -45,8 +45,8 @@ func decodeControl(text string, value any) error { func parseControl(s *Stage, fields map[string]string) error { if text, ok := fields["resources"]; ok { - if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !ValidResourceKey(s.Resources.Exclusive) { - return fmt.Errorf("resources must contain exactly one exclusive key (1–128 ASCII letters, digits, _ or -; start with a letter)") + if err := decodeControl(text, &s.Resources); err != nil || s.Resources == nil || !validStageResources(*s.Resources) { + return fmt.Errorf("resources must contain exactly one exclusive or pool key (1–128 ASCII letters, digits, _ or -; start with a letter)") } delete(fields, "resources") } @@ -81,6 +81,11 @@ func parseControl(s *Stage, fields map[string]string) error { return nil } +func validStageResources(resources protocol.StageResources) bool { + return (ValidResourceKey(resources.Exclusive) && resources.Pool == "") || + (resources.Exclusive == "" && ValidResourceKey(resources.Pool)) +} + func validateControl(s Stage, stages map[string]bool) error { if s.Resources != nil && s.Type == "human" { return fmt.Errorf("stage %s: human stages cannot declare resources", s.Key) diff --git a/internal/pipeline/resources.go b/internal/pipeline/resources.go index 47cef572..61a1f3a6 100644 --- a/internal/pipeline/resources.go +++ b/internal/pipeline/resources.go @@ -200,6 +200,7 @@ type ResourceLease struct { const maxPoolMembers = 32 const maxTargetMetadata = 16 +const maxTargetMetadataKey = 128 const maxTargetValue = 128 func validateTargetMetadata(metadata map[string]string) error { @@ -207,7 +208,7 @@ func validateTargetMetadata(metadata map[string]string) error { return fmt.Errorf("too many target metadata fields") } for k, v := range metadata { - if !identifier.MatchString(k) || len(v) > maxTargetValue || !utf8.ValidString(v) { + if !identifier.MatchString(k) || len(k) > maxTargetMetadataKey || len(v) > maxTargetValue || !utf8.ValidString(v) { return fmt.Errorf("invalid target metadata") } } @@ -330,6 +331,29 @@ func (s *Store) RequestResourcePool(ctx context.Context, pool string, visit Reso if err != sql.ErrNoRows { return ResourceLease{}, nil, nil, err } + // A queued visit may be re-entered after the process that owned its live + // channel was restarted. Reattach to the durable waiter instead of trying + // to insert a second reservation. + var persistedPool string + err = tx.QueryRowContext(ctx, "SELECT pool_name FROM resource_waiters WHERE run_id=? AND stage_key=? AND sequence=?", visit.RunID, visit.Stage, visit.Sequence).Scan(&persistedPool) + if err == nil { + if persistedPool != pool { + return ResourceLease{}, nil, nil, fmt.Errorf("resource waiter pool visit is stale") + } + wait, waitErr := s.resourceWait(ctx, visit.RunID) + if waitErr != nil { + return ResourceLease{}, nil, nil, waitErr + } + if s.resourceWaiters == nil { + s.resourceWaiters = make(map[ResourceVisit]chan ResourceLease) + } + ch := make(chan ResourceLease, 1) + s.resourceWaiters[visit] = ch + return ResourceLease{}, ch, wait, nil + } + if err != sql.ErrNoRows { + return ResourceLease{}, nil, nil, err + } members, err := s.resourcePool(ctx, tx, pool) if err != nil { return ResourceLease{}, nil, nil, fmt.Errorf("unknown resource pool: %w", err) @@ -629,7 +653,7 @@ func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource stri } found := false for _, member := range members { - if member.Member != resource || !member.Enabled || member.Quarantined { + if !member.Enabled || member.Quarantined { continue } var count int @@ -638,8 +662,8 @@ func (s *Store) handoffResourceTx(ctx context.Context, tx *sql.Tx, resource stri } if count == 0 { selected, found = poolTarget(member), true + break } - break } if !found { continue diff --git a/internal/pipeline/resources_test.go b/internal/pipeline/resources_test.go index ac4980d7..90e44365 100644 --- a/internal/pipeline/resources_test.go +++ b/internal/pipeline/resources_test.go @@ -39,6 +39,19 @@ func TestResourceDeclaration(t *testing.T) { } } +func TestResourcePoolDeclaration(t *testing.T) { + base := Template([]string{"work"}, true) + graph, err := Parse(base + "```resources\n{\"pool\":\"android-targets\"}\n```\n") + if err != nil || graph.Stages[0].Resources == nil || graph.Stages[0].Resources.Pool != "android-targets" { + t.Fatalf("pool declaration: %#v, %v", graph.Stages[0].Resources, err) + } + for _, value := range []string{`{"exclusive":"phone","pool":"targets"}`, `{"pool":""}`, `{"pool":"a b"}`} { + if _, err := Parse(base + "```resources\n" + value + "\n```\n"); err == nil { + t.Fatalf("accepted invalid pool declaration %s", value) + } + } +} + func TestResourcePoolAllocatesConcreteMembersAndExcludesDirectOwner(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() @@ -95,6 +108,41 @@ func TestResourcePoolFIFOWakeAndQuarantineIsolation(t *testing.T) { releaseTestResource(t, s, held) } +func TestResourcePoolHandoffUsesAnotherMemberWhenReleasedTargetQuarantined(t *testing.T) { + s, create := resourceStore(t) + ctx := context.Background() + pool := protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ + {Member: "phone", Enabled: true}, {Member: "emulator", Enabled: false}, + }} + if err := s.ConfigureResourcePool(ctx, pool); err != nil { + t.Fatal(err) + } + owner, ready, _, err := s.RequestResourcePool(ctx, pool.Name, create(pool.Name)) + if err != nil || ready != nil || owner.Target.Member != "phone" { + t.Fatalf("owner: %+v %v", owner, err) + } + _, waiter, _, err := s.RequestResourcePool(ctx, pool.Name, create(pool.Name)) + if err != nil || waiter == nil { + t.Fatalf("waiter: %v", err) + } + pool.Members[0].Quarantined = true + pool.Members[1].Enabled = true + if err := s.ConfigureResourcePool(ctx, pool); err != nil { + t.Fatal(err) + } + releaseTestResource(t, s, owner) + select { + case next := <-waiter: + if next.Target == nil || next.Target.Member != "emulator" { + t.Fatalf("wrong target: %+v", next.Target) + } + s.AcceptResource(next) + releaseTestResource(t, s, next) + case <-time.After(time.Second): + t.Fatal("quarantined released phone blocked the free emulator") + } +} + func TestResourcePoolRejectsStaleVisitForExistingOwner(t *testing.T) { s, create := resourceStore(t) ctx := context.Background() @@ -227,6 +275,7 @@ func TestResourcePoolRejectsInvalidDeclarations(t *testing.T) { {Name: "duplicate", Members: []protocol.ResourcePoolMember{{Member: "phone"}, {Member: "phone"}}}, {Name: "bad-member", Members: []protocol.ResourcePoolMember{{Member: "not valid"}}}, {Name: "bad-metadata", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{"bad key": "x"}}}}, + {Name: "oversized-metadata-key", Members: []protocol.ResourcePoolMember{{Member: "phone", Metadata: map[string]string{strings.Repeat("k", maxTargetMetadataKey+1): "x"}}}}, } for _, pool := range cases { if err := s.ConfigureResourcePool(ctx, pool); err == nil { diff --git a/internal/protocol/message.go b/internal/protocol/message.go index dda3c4a7..b5359fbb 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -41,6 +41,7 @@ type Stage struct { // StageResources declares one capacity-one resource, local to this store authority. type StageResources struct { Exclusive string `json:"exclusive"` + Pool string `json:"pool"` } // Limits are enforced by the runner and never included in the model prompt. diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 26828802..ccbcc47d 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -331,11 +331,12 @@ type MemoryRead struct { } type ControlEvidence struct { - Visit int `json:"visit"` - MaxVisits int `json:"max_visits"` - LimitReached bool `json:"limit_reached,omitempty"` - ModelTarget string `json:"model_target,omitempty"` - Memory map[string]MemoryRead `json:"memory,omitempty"` + Visit int `json:"visit"` + MaxVisits int `json:"max_visits"` + LimitReached bool `json:"limit_reached,omitempty"` + ModelTarget string `json:"model_target,omitempty"` + Memory map[string]MemoryRead `json:"memory,omitempty"` + ResourceTarget *ResourceTarget `json:"resource_target,omitempty"` } // StageFileChanges is observed workspace evidence, not a model's claim of authorship. diff --git a/internal/server/resources.go b/internal/server/resources.go index ecb5b7a5..4e6da960 100644 --- a/internal/server/resources.go +++ b/internal/server/resources.go @@ -11,7 +11,19 @@ import ( ) func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) { - lease, ready, wait, err := r.store.RequestResource(ctx, visit) + return r.acquireResourceWithPool(ctx, visit, "", workspace, completed) +} + +func (r *runner) acquireResourceWithPool(ctx context.Context, visit pipeline.ResourceVisit, pool string, workspace string, completed int) (pipeline.ResourceLease, error) { + var lease pipeline.ResourceLease + var ready <-chan pipeline.ResourceLease + var wait *protocol.ResourceWait + var err error + if pool != "" { + lease, ready, wait, err = r.store.RequestResourcePool(ctx, pool, visit) + } else { + lease, ready, wait, err = r.store.RequestResource(ctx, visit) + } if err != nil { return lease, err } diff --git a/internal/server/resources_test.go b/internal/server/resources_test.go index 837560eb..62a91ec8 100644 --- a/internal/server/resources_test.go +++ b/internal/server/resources_test.go @@ -137,6 +137,97 @@ func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) { } } +func TestResourcePoolRunnerHandoffUsesConcreteTargetAfterQueue(t *testing.T) { + r, events, _ := resourceRunner(t) + ctx := context.Background() + source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"pool\":\"targets\"}\n```\n" + if _, err := r.store.Submit(ctx, "pool-runner", source); err != nil { + t.Fatal(err) + } + items, err := r.store.List(ctx) + if err != nil || len(items) != 1 { + t.Fatal("pool definition missing", err) + } + definition, err := r.store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + if err := r.store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{ + {Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "phone-serial", "type": "phone"}}, + {Member: "emulator", Enabled: true, Metadata: map[string]string{"serial": "emu-serial", "type": "emulator"}}, + }}); err != nil { + t.Fatal(err) + } + ownerRun, _, err := r.store.CreateRun(ctx, definition, "phone-owner", t.TempDir(), false, nil, nil) + if err != nil { + t.Fatal(err) + } + phone, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "phone", RunID: ownerRun, Stage: "work", Sequence: 1}) + if err != nil || ch != nil { + t.Fatalf("phone owner acquire: channel=%v err=%v", ch, err) + } + defer r.releaseResource(phone, true) + + aStarted, releaseA := make(chan struct{}), make(chan struct{}) + bRan := make(chan *protocol.ResourceTarget, 1) + r.execute = func(stageCtx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { + target, ok := stage.ResourceTarget(stageCtx) + if !ok || target == nil { + return stage.Result{}, errors.New("missing resource target context") + } + if input["prompt"] == "A" { + if target.Member != "emulator" || target.Metadata["serial"] != "emu-serial" || target.Metadata["type"] != "emulator" { + return stage.Result{}, fmt.Errorf("wrong first target: %#v", target) + } + close(aStarted) + <-releaseA + } else { + if deadline, ok := stageCtx.Deadline(); !ok || time.Until(deadline) < 15*time.Millisecond { + return stage.Result{}, errors.New("execution deadline was consumed in pool queue") + } + bRan <- target + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + start := func(key, prompt string) *protocol.RunDetail { + run, err := r.start(protocol.Message{PipelineID: definition.ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}}) + if err != nil { + t.Fatal(err) + } + return run + } + a := start("pool-a", "A") + <-aStarted + b := start("pool-b", "B") + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) + time.Sleep(80 * time.Millisecond) + select { + case <-bRan: + t.Fatal("pool waiter ran before either member was released") + default: + } + if err := r.releaseResource(phone, true); err != nil { + t.Fatal(err) + } + var target *protocol.ResourceTarget + select { + case target = <-bRan: + case <-time.After(3 * time.Second): + t.Fatal("pool waiter did not resume") + } + if target.Member != "phone" || target.Metadata["serial"] != "phone-serial" { + t.Fatalf("wrong queued target: %#v", target) + } + awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + close(releaseA) + awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" }) + r.wg.Wait() + detail, err := r.store.GetRun(ctx, b.ID) + if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Control.ResourceTarget == nil { + t.Fatalf("pool target evidence missing: detail=%#v err=%v", detail, err) + } +} + func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) { r, events, start := resourceRunner(t) started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{}) diff --git a/internal/server/runner.go b/internal/server/runner.go index 0ee8ec2c..ec5f1509 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -1241,11 +1241,16 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na _, declaresSkip := def.Edges["skipped"] skip := !isHuman && declaresSkip && !state.uiSurface if selected.Resources != nil && !skip { - if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) { + if isHuman || (selected.Resources.Exclusive == "" && selected.Resources.Pool == "") || (selected.Resources.Exclusive != "" && selected.Resources.Pool != "") || (selected.Resources.Exclusive != "" && !pipeline.ValidResourceKey(selected.Resources.Exclusive)) || (selected.Resources.Pool != "" && !pipeline.ValidResourceKey(selected.Resources.Pool)) { finish("failed", "error", "invalid persisted resource declaration") return } - lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed) + resource := selected.Resources.Exclusive + pool := selected.Resources.Pool + if pool != "" { + resource = pool + } + lease, err := r.acquireResourceWithPool(ctx, pipeline.ResourceVisit{Resource: resource, RunID: id, Stage: key, Sequence: sequence}, pool, workspace, state.completed) if err != nil { if ctx.Err() != nil { finish("canceled", "", err.Error()) @@ -1392,6 +1397,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na var piClient *stage.PiClient var err error stageCtx := ctx + if held != nil && held.Target != nil { + stageCtx = stage.WithResourceTarget(stageCtx, held.Target) + } var modelConfig stage.ModelConfig var budgetBefore stage.BudgetSnapshot if stage.EffectiveType(def.Type) == stage.ModelType { @@ -1408,7 +1416,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na // its cumulative usage before the visit and record only this visit's // increment on the step, never the cumulative total again. budgetBefore = budgets[key].Snapshot() - stageCtx = stage.WithBudget(ctx, budgets[key]) + stageCtx = stage.WithBudget(stageCtx, budgets[key]) } snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second) before := filediff.Capture(snapshotCtx, workspace) @@ -1470,7 +1478,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input) client.Transcript = plain.FormatAudit } else { - result, err = stage.DefaultRegistry().Run(ctx, def, input) + result, err = stage.DefaultRegistry().Run(stageCtx, def, input) } }() if held != nil { @@ -1571,6 +1579,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } step.Target = def.Edges[step.Outcome] step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads} + if held != nil && held.Target != nil { + step.Control.ResourceTarget = held.Target + } if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits { for _, outcome := range selected.Limits.On { if outcome == step.Outcome { diff --git a/internal/stage/stage.go b/internal/stage/stage.go index 0e6b7af9..ca6d1e06 100644 --- a/internal/stage/stage.go +++ b/internal/stage/stage.go @@ -5,8 +5,34 @@ package stage import ( "context" "fmt" + + "git.a-wan.net/alan/pipelines/internal/protocol" ) +type resourceTargetContextKey struct{} + +// WithResourceTarget attaches the server-granted concrete target to a stage's +// runtime context. It is deliberately separate from model input and contains +// no capability or credential material. +func WithResourceTarget(ctx context.Context, target *protocol.ResourceTarget) context.Context { + if target == nil { + return ctx + } + copy := &protocol.ResourceTarget{Member: target.Member} + if target.Metadata != nil { + copy.Metadata = map[string]string{} + for key, value := range target.Metadata { + copy.Metadata[key] = value + } + } + return context.WithValue(ctx, resourceTargetContextKey{}, copy) +} + +func ResourceTarget(ctx context.Context) (*protocol.ResourceTarget, bool) { + target, ok := ctx.Value(resourceTargetContextKey{}).(*protocol.ResourceTarget) + return target, ok +} + // Values are runtime data, not the prose output contract in a definition. type Values map[string]any