Remove resource pools, leases and wait queues

Drop the resource allocation feature end to end: store tables and migration, runner scheduling and cleanup, protocol types and commands, CLI commands, TUI screens, tests and docs. Existing databases open cleanly and definitions carrying a retired resources block still validate.
This commit is contained in:
2026-10-19 22:27:27 +01:00
parent a676b30ff0
commit 72aecd8b1f
45 changed files with 282 additions and 3954 deletions
-511
View File
@@ -1,511 +0,0 @@
package server
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/client"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
type fixedRemoteListener struct{ net.Listener }
func (l fixedRemoteListener) Accept() (net.Conn, error) {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return fixedRemoteConn{Conn: conn}, nil
}
type fixedRemoteConn struct{ net.Conn }
func (fixedRemoteConn) RemoteAddr() net.Addr {
return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443}
}
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
// executor. It never calls inference, a phone, or a deployed server.
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json"))
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n"
if _, err := store.Submit(ctx, "scratch-resource", source); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store).(*runHandler)
started, release := make(chan struct{}), make(chan struct{})
handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
if input["prompt"] == "A" {
close(started)
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
}
return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil
}
handler.runner.events.interval = time.Millisecond
events := make(chan protocol.RunEventFrame, 64)
sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error {
if m.RunEvent != nil {
events <- *m.RunEvent
}
return nil
})
sub.ready()
defer sub.close()
srv := httptest.NewServer(handler)
defer srv.Close()
defer func() { cancel(); WaitRuns(handler) }()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
run := func(args ...string) string {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
}
t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String())
return out.String()
}
parseRun := func(output string) int64 {
var m protocol.Message
if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil {
t.Fatal("CLI run response", output, err)
}
return m.RunDetail.ID
}
run("validate", "testdata/android-resource-draft.md", "--json")
a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a"))
<-started
b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b"))
awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
var detail protocol.RunDetail
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 {
t.Fatal("CLI did not observe queue", err)
}
if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") {
t.Fatal("list omitted resource status", output)
}
var snapshot *protocol.RunEventFrame
handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error {
if m.RunEvent != nil {
snapshot = m.RunEvent
}
return nil
})
handler.runner.events.unsubscribeClient("snapshot-evidence")
if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet {
t.Fatal("subscription snapshot omitted resource wait")
}
close(release)
run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json")
detail = protocol.RunDetail{}
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 {
t.Fatal("automatic progress not observed", err)
}
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
}
func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"
if _, err := store.Submit(ctx, "operator-resource", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) string {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
}
return out.String()
}
inspection := runCLI("resource-inspect", visit.Resource, "--json")
if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") {
t.Fatalf("inspection leaked capability: %s", inspection)
}
runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json")
inspection = runCLI("resource-inspect", visit.Resource, "--json")
var recovered protocol.ResourceInspection
if err := json.Unmarshal([]byte(inspection), &recovered); err != nil {
t.Fatal(err)
}
if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" {
t.Fatalf("CLI recovery state was not persisted: %s", inspection)
}
}
func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
lease, ch, _, err := store.RequestResource(ctx, visit)
if err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) error {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
err := app.Run(ctx, append([]string{"--url", endpoint}, args...))
if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") {
t.Fatalf("private capability appeared in CLI output: %s", out.String())
}
return err
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted wrong owner")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale stage")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale sequence")
}
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
t.Fatal(err)
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted non-quarantined owner")
}
inspection, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 {
t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits))
}
}
func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) {
for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} {
if isLoopbackPeer(remote) {
t.Fatalf("accepted non-loopback peer %q", remote)
}
}
for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} {
if !isLoopbackPeer(remote) {
t.Fatalf("rejected loopback peer %q", remote)
}
}
// X-Forwarded-For is deliberately not consulted: a proxy-supplied header
// cannot turn a remote WebSocket peer into the local operator transport.
}
func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
before, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
server := httptest.NewUnstartedServer(Handler(ctx, store))
server.Listener = fixedRemoteListener{Listener: server.Listener}
server.Start()
defer server.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}})
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
var welcome protocol.Message
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
t.Fatal(err)
}
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil {
t.Fatal(err)
}
var refused protocol.Message
if err := wsjson.Read(ctx, conn, &refused); err != nil {
t.Fatal(err)
}
if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") {
t.Fatalf("forged peer was accepted: %#v", refused)
}
after, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(before, after) {
t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after)
}
trusted := httptest.NewServer(Handler(ctx, store))
defer trusted.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect")
if err != nil {
t.Fatal(err)
}
defer operator.Close()
if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil {
t.Fatalf("trusted operator transport refused inspection: %v", err)
}
}
func TestResourcePoolOperatorCLIConfigureReadbackAndPreserveLease(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
dbPath := filepath.Join(t.TempDir(), "pool-operator.db")
store, err := pipeline.Open(dbPath)
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "old"}}}}); err != nil {
t.Fatal(err)
}
if _, err := store.Submit(ctx, "pool-owner", pipeline.Template([]string{"work"}, true)+"```resources\n{\"pool\":\"targets\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil || len(items) != 1 {
t.Fatal("pool pipeline missing", err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "pool-owner", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
lease, ready, _, err := store.RequestResourcePool(ctx, "targets", pipeline.ResourceVisit{Resource: "targets", RunID: runID, Stage: "work", Sequence: 1})
if err != nil || ready != nil {
t.Fatalf("pool lease: ready=%v err=%v", ready, err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
normal, err := client.Open(ctx, endpoint)
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.ConfigureResourcePool, ResourcePool: &protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true}}}}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("remote configuration was accepted: %v", err)
}
normal.Close()
declaration := filepath.Join(t.TempDir(), "pool.json")
if err := os.WriteFile(declaration, []byte(`{"name":"targets","members":[{"member":"phone","enabled":true,"metadata":{"serial":"new"}},{"member":"emulator","enabled":false}]}`), 0600); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
for _, fromStdin := range []bool{false, true} {
oversized := strings.Repeat(" ", 256*1024+1)
path := filepath.Join(t.TempDir(), "oversized.json")
if err := os.WriteFile(path, []byte(oversized), 0600); err != nil {
t.Fatal(err)
}
if fromStdin {
path = "-"
}
bounded := cli.App{Out: &out, Err: &out, In: strings.NewReader(oversized)}
err := bounded.Run(ctx, []string{"--url", endpoint, "resource-pool-configure", path, "--yes", "--json"})
if err == nil || !strings.Contains(err.Error(), "exceeds 262144 bytes") {
t.Fatalf("oversized pool input (stdin=%v): %v", fromStdin, err)
}
out.Reset()
}
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http"), "resource-pool-configure", declaration, "--yes", "--json"}); err != nil {
t.Fatalf("configure CLI: %v\n%s", err, out.String())
}
out.Reset()
app = cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http"), "resource-pool-inspect", "targets", "--json"}); err != nil {
t.Fatalf("inspect CLI: %v\n%s", err, out.String())
}
var pool protocol.ResourcePool
if err := json.Unmarshal(out.Bytes(), &pool); err != nil || len(pool.Members) != 2 || pool.Members[0].Metadata["serial"] != "new" {
t.Fatalf("bad pool readback: %s (%v)", out.String(), err)
}
inspection, err := store.InspectResource(ctx, "phone")
if err != nil || inspection.Owner == nil || inspection.Owner.RunID != runID {
t.Fatalf("configuration released active lease: %#v (%v)", inspection, err)
}
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
t.Fatal(err)
}
if err := store.ReleaseResource(ctx, lease); err != nil {
t.Fatal(err)
}
srv.Close()
store.Close()
reopened, err := pipeline.Open(dbPath)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
persisted, err := reopened.InspectResourcePool(ctx, "targets")
if err != nil || persisted.Members[0].Metadata["serial"] != "new" {
t.Fatalf("pool did not survive reopen: %#v (%v)", persisted, err)
}
}
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
baseline, err := os.ReadFile("testdata/android-v3-published.md")
if err != nil {
t.Fatal(err)
}
if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" {
t.Fatal("baseline is not retrieved Android v3", hash)
}
draft, err := os.ReadFile("testdata/android-resource-draft.md")
if err != nil {
t.Fatal(err)
}
old, err := pipeline.Parse(string(baseline))
if err != nil {
t.Fatal(err)
}
updated, err := pipeline.Parse(string(draft))
if err != nil {
t.Fatal(err)
}
if len(old.Stages) != len(updated.Stages) {
t.Fatal("draft changed graph shape")
}
for i := range old.Stages {
a, b := old.Stages[i], updated.Stages[i]
if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] {
t.Fatal("draft changed approved model")
}
if b.Key == "ui_reviewer" {
if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" {
t.Fatal("reviewer resource missing")
}
} else if b.Resources != nil {
t.Fatal("non-UI stage acquired phone")
}
a.Prompt, b.Prompt = "", ""
a.Resources, b.Resources = nil, nil
if !reflect.DeepEqual(a, b) {
t.Fatal("draft changed fields outside approved role split/resource", a.Key)
}
}
}
-72
View File
@@ -1,72 +0,0 @@
package server
import (
"context"
"errors"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) {
return r.acquireResourceWithPool(ctx, visit, "", workspace, completed)
}
func (r *runner) acquireResourceWithPool(ctx context.Context, visit pipeline.ResourceVisit, pool string, workspace string, completed int) (pipeline.ResourceLease, error) {
var lease pipeline.ResourceLease
var ready <-chan pipeline.ResourceLease
var wait *protocol.ResourceWait
var err error
if pool != "" {
lease, ready, wait, err = r.store.RequestResourcePool(ctx, pool, visit)
} else {
lease, ready, wait, err = r.store.RequestResource(ctx, visit)
}
if err != nil {
return lease, err
}
if ready != nil {
r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, Status: protocol.RunStatusQueued, CurrentStage: visit.Stage, CompletedSteps: completed, ResourceWaitSet: true, ResourceWait: wait}, false)
select {
case lease = <-ready:
r.store.AcceptResource(lease)
case <-ctx.Done():
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.CancelResource(cleanup, visit); err != nil {
return lease, errors.Join(ctx.Err(), err)
}
return lease, ctx.Err()
}
}
if ctx.Err() != nil {
err := r.releaseResource(lease, true) // Never launched, including a concurrent grant.
return pipeline.ResourceLease{}, errors.Join(ctx.Err(), err)
}
r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, ResourceWaitSet: true}, false)
return lease, nil
}
func (r *runner) releaseResource(lease pipeline.ResourceLease, confirmed bool) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.SetResourceCleanup(ctx, lease, confirmed); err != nil {
return err
}
if confirmed {
return r.store.ReleaseResource(ctx, lease)
}
return nil
}
func (r *runner) resourceQuiescent(id int64, executionErr error, panicked bool) bool {
if panicked || errors.Is(executionErr, stage.ErrStageCleanup) {
return false
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
sessions, err := r.store.ActiveAgentSessionsForRun(ctx, id)
return err == nil && len(sessions) == 0
}
-418
View File
@@ -1,418 +0,0 @@
package server
import (
"context"
"database/sql"
"errors"
"fmt"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db"))
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
r.events = newRunEventHub()
r.events.interval = time.Millisecond
events := make(chan protocol.RunEventFrame, 128)
sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error {
if m.RunEvent != nil {
events <- *m.RunEvent
}
return nil
})
sub.ready()
t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() })
count := 0
return r, events, func(resource, timeout, prompt string) *protocol.RunDetail {
count++
source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1)
if resource != "" {
source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n"
}
name := fmt.Sprint("resource-", count)
if _, err := s.Submit(ctx, name, source); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
var pid int64
for _, p := range items {
if p.Name == name {
pid = p.ID
}
}
d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
if err != nil {
t.Fatal(err)
}
return d
}
}
func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame {
t.Helper()
timer := time.NewTimer(3 * time.Second)
defer timer.Stop()
for {
select {
case event := <-events:
if event.RunID == id && predicate(event) {
return event
}
case <-timer.C:
t.Fatalf("resource event missing for run %d", id)
return protocol.RunEventFrame{}
}
}
}
func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) {
r, events, start := resourceRunner(t)
aStarted, releaseA := make(chan struct{}), make(chan struct{})
bStarted := make(chan struct{}, 1)
var bCalls atomic.Int32
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
switch input["prompt"] {
case "A":
close(aStarted)
select {
case <-releaseA:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
case "B":
bCalls.Add(1)
deadline, ok := ctx.Deadline()
if !ok || time.Until(deadline) < 15*time.Millisecond {
return stage.Result{}, errors.New("execution timeout was consumed in queue")
}
bStarted <- struct{}{}
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "2s", "A")
<-aStarted
b := start("X", "30ms", "B")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
<-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout.
d, err := r.store.GetRun(context.Background(), b.ID)
if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 {
t.Fatal("queued execution consumed accounting", d, err, bCalls.Load())
}
if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil {
t.Fatal("resource wait exposed a human decision")
}
c := start("Y", "1s", "C")
awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
select {
case <-bStarted:
t.Fatal("B ran while A held X")
default:
}
owner, err := r.store.GetRun(context.Background(), a.ID)
if err != nil || owner.Status != "running" {
t.Fatal("A did not remain owner", err)
}
close(releaseA)
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
r.wg.Wait()
d, err = r.store.GetRun(context.Background(), b.ID)
if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 {
t.Fatal("resume did not execute exactly one fresh visit", d, err)
}
}
func TestResourcePoolRunnerHandoffUsesConcreteTargetAfterQueue(t *testing.T) {
r, events, _ := resourceRunner(t)
ctx := context.Background()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"pool\":\"targets\"}\n```\n"
if _, err := r.store.Submit(ctx, "pool-runner", source); err != nil {
t.Fatal(err)
}
items, err := r.store.List(ctx)
if err != nil || len(items) != 1 {
t.Fatal("pool definition missing", err)
}
definition, err := r.store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
if err := r.store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "phone-serial", "type": "phone"}},
{Member: "emulator", Enabled: true, Metadata: map[string]string{"serial": "emu-serial", "type": "emulator"}},
}}); err != nil {
t.Fatal(err)
}
ownerRun, _, err := r.store.CreateRun(ctx, definition, "phone-owner", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
phone, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "phone", RunID: ownerRun, Stage: "work", Sequence: 1})
if err != nil || ch != nil {
t.Fatalf("phone owner acquire: channel=%v err=%v", ch, err)
}
defer r.releaseResource(phone, true)
aStarted, releaseA := make(chan struct{}), make(chan struct{})
bRan := make(chan *protocol.ResourceTarget, 1)
r.execute = func(stageCtx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
target, ok := stage.ResourceTarget(stageCtx)
if !ok || target == nil {
return stage.Result{}, errors.New("missing resource target context")
}
if input["prompt"] == "A" {
if target.Member != "emulator" || target.Metadata["serial"] != "emu-serial" || target.Metadata["type"] != "emulator" {
return stage.Result{}, fmt.Errorf("wrong first target: %#v", target)
}
close(aStarted)
<-releaseA
} else {
if deadline, ok := stageCtx.Deadline(); !ok || time.Until(deadline) < 15*time.Millisecond {
return stage.Result{}, errors.New("execution deadline was consumed in pool queue")
}
bRan <- target
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
start := func(key, prompt string) *protocol.RunDetail {
run, err := r.start(protocol.Message{PipelineID: definition.ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
if err != nil {
t.Fatal(err)
}
return run
}
a := start("pool-a", "A")
<-aStarted
b := start("pool-b", "B")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
time.Sleep(80 * time.Millisecond)
select {
case <-bRan:
t.Fatal("pool waiter ran before either member was released")
default:
}
if err := r.releaseResource(phone, true); err != nil {
t.Fatal(err)
}
var target *protocol.ResourceTarget
select {
case target = <-bRan:
case <-time.After(3 * time.Second):
t.Fatal("pool waiter did not resume")
}
if target.Member != "phone" || target.Metadata["serial"] != "phone-serial" {
t.Fatalf("wrong queued target: %#v", target)
}
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
close(releaseA)
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
r.wg.Wait()
detail, err := r.store.GetRun(ctx, b.ID)
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Control.ResourceTarget == nil {
t.Fatalf("pool target evidence missing: detail=%#v err=%v", detail, err)
}
}
func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) {
r, events, start := resourceRunner(t)
started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{})
var waitingCalls atomic.Int32
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
if input["prompt"] == "owner" {
close(started)
<-ctx.Done()
close(cleaning)
<-cleanupDone
return stage.Result{}, ctx.Err()
}
waitingCalls.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "2s", "owner")
<-started
b := start("X", "30ms", "waiter")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
if err := r.cancel(a.ID); err != nil {
t.Fatal(err)
}
<-cleaning
select {
case <-time.After(50 * time.Millisecond):
if waitingCalls.Load() != 0 {
t.Fatal("handoff before cleanup")
}
}
close(cleanupDone)
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
if waitingCalls.Load() != 1 {
t.Fatal("waiter did not resume exactly once")
}
}
func TestResourceQuarantineAndQueuedCancellation(t *testing.T) {
for _, failure := range []string{"cleanup", "panic", "restoration"} {
t.Run(failure, func(t *testing.T) {
r, events, start := resourceRunner(t)
started, release := make(chan struct{}), make(chan struct{})
var calls atomic.Int32
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
if input["prompt"] == "owner" {
close(started)
select {
case <-release:
case <-ctx.Done():
}
if failure == "panic" {
panic("unknown cleanup")
}
if failure == "restoration" {
return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil
}
return stage.Result{}, stage.ErrStageCleanup
}
calls.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "1s", "owner")
<-started
b := start("X", "30ms", "waiter")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
close(release)
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
if err := r.cancel(b.ID); err != nil {
t.Fatal(err)
}
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool {
return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil
})
r.wg.Wait()
if calls.Load() != 0 {
t.Fatal("quarantined resource handed off")
}
exists, err := r.store.HasResourceOwner(context.Background(), a.ID)
if err != nil || !exists {
t.Fatal("quarantine evidence lost", err)
}
d, err := r.store.GetRun(context.Background(), b.ID)
if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 {
t.Fatal("canceled queue consumed step", err)
}
})
}
}
func TestResourceEventClearCoalescing(t *testing.T) {
frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}}
mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true})
if !frame.ResourceWaitSet || frame.ResourceWait != nil {
t.Fatal("coalescer retained stale resource wait")
}
}
func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) {
for _, failure := range []string{"begin", "persist"} {
t.Run(failure, func(t *testing.T) {
r, events, start := resourceRunner(t)
db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db"))
if err != nil {
t.Fatal(err)
}
defer db.Close()
trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END`
if failure == "persist" {
trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END`
}
if _, err := db.Exec(trigger); err != nil {
t.Fatal(err)
}
var calls atomic.Int32
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
calls.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "1s", "fail")
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
r.wg.Wait()
if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists {
t.Fatal("safe failure stranded resource", err)
}
if failure == "begin" && calls.Load() != 0 {
t.Fatal("BeginStep failure executed work")
}
b := start("X", "1s", "pass")
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
})
}
}
func TestResourceCleanupQueryFailureQuarantines(t *testing.T) {
r, events, start := resourceRunner(t)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
if err := r.store.Close(); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
a := start("X", "1s", "query-failure")
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
r.wg.Wait()
reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db"))
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
if err := reopened.RecoverRuns(context.Background()); err != nil {
t.Fatal(err)
}
if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists {
t.Fatal("query failure released uncertain owner", err)
}
}
func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) {
r, events, _ := resourceRunner(t)
ctx := context.Background()
source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n"
if _, err := r.store.Submit(ctx, "skip", source); err != nil {
t.Fatal(err)
}
items, _ := r.store.List(ctx)
d, _ := r.store.Get(ctx, items[0].ID, 1)
owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1})
if err != nil || ch != nil {
t.Fatal("owner fixture failed", err)
}
defer r.releaseResource(lease, true)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("skipped stage executed")
return stage.Result{}, nil
}
run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}})
if err != nil {
t.Fatal(err)
}
awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
detail, err := r.store.GetRun(ctx, run.ID)
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" {
t.Fatal("skip not preserved", detail, err)
}
}
+12 -17
View File
@@ -424,9 +424,6 @@ func (s *runSubscription) close() {
}
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.ResourceWaitSet {
into.ResourceWaitSet, into.ResourceWait = true, in.ResourceWait
}
if in.Workspace != "" {
into.Workspace = in.Workspace
}
@@ -503,20 +500,18 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
ResourceWaitSet: true,
ResourceWait: summary.ResourceWait,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
+15 -83
View File
@@ -86,10 +86,6 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
}
cancel()
if err != nil {
if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource {
log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID)
continue
}
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
}
if err := s.ConfirmRecoveredSession(ctx, owner); err != nil {
@@ -224,9 +220,6 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
// workspace) must not block new work.
for _, owner := range owners {
if _, ok := r.active[owner.RunID]; !ok {
if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace {
continue
}
r.mu.Unlock()
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
}
@@ -452,7 +445,7 @@ func retryInputOverride(base map[string]any, override map[string]any) (map[strin
// retryStageContractEqual compares the complete persisted stage contract. A
// retry may carry completed output across a published version only when the
// stages that produced that output are unchanged; comparing only effective
// types would allow prompt, config, edge, resource, or limit changes through.
// types would allow prompt, config, edge, or limit changes through.
func retryStageContractEqual(left, right protocol.Stage) bool {
a, err := json.Marshal(left)
if err != nil {
@@ -1182,14 +1175,13 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
// Push the terminal delta before the attention notification, so a slow ntfy
// never delays the live update the app folds.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: status,
ResourceWaitSet: true,
Outcome: outcome,
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
RunID: id,
Workspace: workspace,
Status: status,
Outcome: outcome,
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
}, true)
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
// The run is terminal: nothing more can notify for it.
@@ -1257,21 +1249,6 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol
}
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
var held *pipeline.ResourceLease
resourceQuiet := true
release := func() error {
if held != nil {
err := r.releaseResource(*held, resourceQuiet)
held = nil
return err
}
return nil
}
defer func() {
if err := release(); err != nil {
log.Printf("resource cleanup run=%d: %v", id, err)
}
}()
finish := func(status, outcome, diagnostic string) {
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
}
@@ -1336,7 +1313,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
maxVisits = selected.Limits.MaxVisits
}
// Validate the next visit without consuming it. An already-exhausted
// stage must not queue for a resource it can never execute.
// stage must not begin work it can never execute.
if !isHuman {
if state.visits[key]+1 > 5 {
finish("failed", "error", "stage visit budget exceeded")
@@ -1349,27 +1326,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
_, declaresSkip := def.Edges["skipped"]
skip := !isHuman && declaresSkip && !state.uiSurface
if selected.Resources != nil && !skip {
if isHuman || (selected.Resources.Exclusive == "" && selected.Resources.Pool == "") || (selected.Resources.Exclusive != "" && selected.Resources.Pool != "") || (selected.Resources.Exclusive != "" && !pipeline.ValidResourceKey(selected.Resources.Exclusive)) || (selected.Resources.Pool != "" && !pipeline.ValidResourceKey(selected.Resources.Pool)) {
finish("failed", "error", "invalid persisted resource declaration")
return
}
resource := selected.Resources.Exclusive
pool := selected.Resources.Pool
if pool != "" {
resource = pool
}
lease, err := r.acquireResourceWithPool(ctx, pipeline.ResourceVisit{Resource: resource, RunID: id, Stage: key, Sequence: sequence}, pool, workspace, state.completed)
if err != nil {
if ctx.Err() != nil {
finish("canceled", "", err.Error())
} else {
finish("failed", "error", err.Error())
}
return
}
held, resourceQuiet = &lease, true
}
state.nextSequence = sequence + 1
state.visits[key]++
// Publish the run's server-captured changed paths into the input memory
@@ -1403,13 +1359,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
// queued -> running: report the new current stage and the step start.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
ResourceWaitSet: true,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
}, false)
if isHuman {
// A human step is O(1): one BeginStep plus one MarkWaiting, no
@@ -1516,9 +1471,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
var piClient *stage.PiClient
var err error
stageCtx := ctx
if held != nil && held.Target != nil {
stageCtx = stage.WithResourceTarget(stageCtx, held.Target)
}
var modelConfig stage.ModelConfig
var budgetBefore stage.BudgetSnapshot
if stage.EffectiveType(def.Type) == stage.ModelType {
@@ -1542,9 +1494,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
snapshotCancel()
var stageSession string
panicked := false
if held != nil {
resourceQuiet = false
}
func() {
defer func() {
if recover() != nil {
@@ -1600,12 +1549,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
}
}()
if held != nil {
resourceQuiet = r.resourceQuiescent(id, err, panicked)
if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true {
resourceQuiet = false
}
}
// Capture after cleanup, even when the stage's context was canceled.
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
after := filediff.Capture(snapshotCtx, workspace, before)
@@ -1698,9 +1641,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
step.Target = def.Edges[step.Outcome]
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
if held != nil && held.Target != nil {
step.Control.ResourceTarget = held.Target
}
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
for _, outcome := range selected.Limits.On {
if outcome == step.Outcome {
@@ -1765,14 +1705,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
return
}
}
if held != nil && !resourceQuiet {
finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection")
return
}
if err := release(); err != nil {
finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error())
return
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
-312
View File
@@ -1,312 +0,0 @@
# stage implementer
```limits
{"max_visits":5,"on":["blocked_retryable"]}
```
```memory
{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"}
```
```output
{ task, acceptance_checks, changed_files, implementation_summary, new_helpers }
```
```prompt
Implement input.prompt in the run directory using the repository's actual languages and
file types; create files as needed.
Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where
the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go
looking for what it already names; if it is wrong or out of date, say so in your output.
Read the surrounding code before editing so you extend the existing helpers and patterns rather than
duplicating them. Keep each change as small as the task allows and follow the repository's own
conventions and error handling.
This is the Android app. Build and test it with the project's own Gradle invocation as the map
gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that
adds a raw value fails the repository's own check. Perform code/build/unit tests only;
no adb, install, capture, device-setting changes or restoring main's build. The UI reviewer
alone verifies rendered surfaces under its declared runner resource lease.
Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new
shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them -
and report the exact commands and observed output. Address any concrete findings the runner supplies
and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not
commit or push; the committer stage lands the work.
```
```tools
read
glob
grep
edit
shell
```
```configs
provider=openai
model=gpt-5.6-terra
retries=1
timeout=30m
```
```edges
success -> reviewer
blocked_retryable -> implementer
blocked -> #failure
error -> #failure
```
# stage reviewer
```limits
{"max_visits": 3, "on": ["rejected"]}
```
```memory
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
```
```output
{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings }
```
```prompt
Verify with builds, unit tests and source review as android/AGENTS.md gives them.
Do not use adb, install, capture, change device settings or restore main's build.
Device verification belongs exclusively to the UI reviewer under its runner resource lease.
Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix finding.
Review the actual changes against the task in input.prompt and the acceptance
checks the implementer recorded. Run the repository's relevant verification checks
with the stage tools (for example build, lint, tests or type checks) and report the
exact commands and observed results in verification_evidence.
Independently assess code quality:
- Duplication: search the run directory for repeated or near-identical functions,
blocks and copy-pasted logic introduced by the change; identify code that should be
shared and record concrete locations in duplication_review.
- Performance and resources: flag redundant computation, unnecessary dependencies,
and repeated I/O or database/filesystem work that wastes time, CPU or memory;
record it in performance_review.
- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for
speed in findings.
Keep verification proportionate to the change: exercise the files and screens the diff
actually touches, capture at most one screenshot per changed surface plus one regression
screen, and do not repeat sweeps or measurements that an earlier visit already recorded.
Return rejected ONLY when at least one must-fix finding exists: a defect that blocks
acceptance and that you can locate concretely (file and line), with evidence and a
remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there
is no must-fix finding but you have non-blocking notes (duplication, performance or
workaround observations), record them in findings and return advisory, which
continues the run without another implementer visit. Return success only when the
observed checks pass and you have nothing further to record. Do not edit files or
claim unexecuted checks.
```
```tools
read
glob
grep
shell
```
```configs
provider=openai
model=gpt-5.6-luna
retries=1
timeout=45m
```
```edges
success -> ui_reviewer
advisory -> ui_reviewer
rejected -> implementer
error -> #failure
```
# stage ui_reviewer
```resources
{"exclusive":"phone-47241FDAS00DLK"}
```
```limits
{"max_visits": 3, "on": ["rejected"]}
```
```memory
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
```
```output
{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings }
```
```prompt
Decide whether this change touches a user-facing UI. If it does not (a backend,
library, CLI, config or data-only change with no rendered surface), set
ui_applicable=false, explain briefly in summary, and return skipped. Do not invent
screenshots or visual work.
When the change does affect a user-facing UI, verify it visually:
Read android/AGENTS.md first. You alone perform phone work while the runner holds
this stage's declared resource lease. Use controlled fixtures for changed states,
including human-decision sheets, on a scratch server/database and throwaway session.
Never mutate an operator session or restart deployed services. Record original
settings and connection/workspace; use the configured authoritative endpoint, not a
hardcoded scratch default. Keep every shell/ADB operation foreground and awaited.
Restore original settings and main's build before successful return, under the same
lease. Report capture/restoration failures explicitly; do not approve incomplete
verification or advertise readiness when cleanup/restoration is uncertain.
Include resource_cleanup_confirmed in output: true only after confirmed restoration;
false on any uncertainty (a release veto that quarantines ownership even if tools are idle).
- Build and launch the application the way the repository actually does it, then
capture screenshots of every relevant screen, state and viewport (default, empty,
loading, error, long content, small and large sizes) with the target platform's
screenshot tooling through shell. Prefer a headless or scripted capture when the
project supports one. Save the images under the run directory and list the exact
capture commands and image paths in screenshots.
- Read the captured images and review them as a UI/UX designer: layout, spacing,
alignment, typographic hierarchy, color and contrast, iconography, interaction
and focus states, loading/empty/error states, accessibility, and consistency.
- Judge the UI against established industry patterns and, above all, the native
conventions of the platform the app targets (operating system, toolkit or web
standards). Record the platform you reviewed against in platform_conventions,
and any deviations and concrete fixes in findings. Infer the target platform from
the task and the repository's actual UI toolkit when it is unstated.
- Approve only when the screenshots show the task and the acceptance checks are met
and the UI is correct, consistent and native to the target platform; otherwise
return rejected with specific, actionable findings the implementer can fix.
Keep the capture pass proportionate: capture each changed state once (for example one
collapsed and one expanded section, one run state, one stuck header) instead of every
screen at multiple viewports or pan directions, and reuse what this run already captured
rather than re-doing it.
If the change is UI-facing but screenshots genuinely cannot be captured
(for example no runnable UI or no display), return blocked rather than approving.
Do not edit files, and do not approve a UI you did not actually see.
Return rejected only for a must-fix visual defect with a concrete location, evidence
and remedy; record non-blocking polish, duplication or convention notes in findings
and return advisory, which continues the run without another implementer visit.
```
```tools
read
glob
grep
shell
```
```configs
provider=openai
model=gpt-5.6-luna
retries=1
timeout=45m
```
```edges
skipped -> committer
approved -> committer
advisory -> committer
rejected -> implementer
blocked -> #failure
error -> #failure
```
# stage committer
```memory
{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"}
```
```output
{ commit, files, conflicts_resolved, summary }
```
```prompt
Commit the work this run produced, and resolve any conflict that is present. This stage
runs after the reviews have approved and is the only stage allowed to commit.
Before committing, check that the change is reflected where it belongs: the packaged
templates, the packaged
authoring skill references. Update anything the change makes stale, and state in the
commit body what you checked, saying explicitly when none of them needs updating.
Stage exactly the files this run changed - the paths in changed_files that still exist -
using the repository's own version control commands, and commit them as one commit whose
subject says what the change does and whose body summarises it. Do not stage unrelated
files, generated or ignored files, or anything another change owns. Do not invent work
that was not done; describe what is actually in the diff.
If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or
cherry-pick left unfinished - resolve each conflict: keep both intents where they are
compatible, and choose the result that satisfies the task where they are not, then
complete the operation so the tree is clean and the commit is finished. Never commit
conflict markers, and never discard the other side's work silently; record what you
resolved.
Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit,
report that instead of creating an empty commit. Report the commit identifier, the files
committed, whatever you resolved, and anything you deliberately left uncommitted.
Finally, bring this branch up to date with main, in this same stage. After committing the work
above, merge main into this branch here, resolve any conflict here - keeping both intents where they
are compatible, and choosing the result that satisfies this task where they are not - run the
repository's checks again after resolving, and commit the completed merge with a message naming what
was combined. The point is that this branch can later be merged back without conflict: leave the
tree clean and the merge finished. Never leave conflict markers, never discard the other side's work
silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of
leaving the merge half-finished.
Then land it: this branch is the work and main is where it belongs, so merge this branch into main
as the last act of this stage. Find the repository's main working tree with `git worktree list` (the
entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a
message saying what the change does. Two lanes can land at the same moment, so take a lock first -
`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when
done - and retry briefly if another lane holds it. Then verify with
`git merge-base --is-ancestor <this branch> main` and report the commit main now has. If main cannot
be updated, say exactly why and leave this branch committed and clean rather than leaving main
half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed
here, it is reported here.
Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer.
```
```tools
read
glob
grep
edit
shell
```
```configs
provider=openai
model=gpt-5.6-terra
retries=1
timeout=30m
```
```edges
success -> #success
blocked -> #failure
error -> #failure
```
-50
View File
@@ -2,9 +2,7 @@ package server
import (
"context"
"fmt"
"log"
"net"
"net/http"
"strings"
"sync"
@@ -270,44 +268,6 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
response.Error = err.Error()
}
}
} else if (request.Type == protocol.InspectResource || request.Type == protocol.ReconcileResource || request.Type == protocol.InspectResourcePool || request.Type == protocol.ConfigureResourcePool) && len(stores) > 0 {
var err error
response.Type = protocol.ResourceInspectionResult
// Recovery has a separate transport path. A stage/model client may
// also be local, so loopback alone is not an authority boundary.
// The CLI uses /operator-connect; the ordinary /connect path is
// deliberately refused even from loopback.
if r.URL.Path != "/operator-connect" || !isLoopbackPeer(r.RemoteAddr) {
response.Type = protocol.Error
response.Error = "resource recovery requires the local operator transport"
} else if request.Type == protocol.InspectResourcePool {
response.Type = protocol.ResourcePoolInspection
var pool protocol.ResourcePool
pool, err = stores[0].InspectResourcePool(ctx, request.Resource)
response.ResourcePool = &pool
} else if request.Type == protocol.ConfigureResourcePool {
response.Type = protocol.ResourcePoolConfigured
if request.ResourcePool == nil {
err = fmt.Errorf("resource pool declaration is required")
} else {
err = stores[0].ConfigureResourcePool(ctx, *request.ResourcePool)
}
} else if request.Type == protocol.InspectResource {
var inspection protocol.ResourceInspection
inspection, err = stores[0].InspectResource(ctx, request.Resource)
response.ResourceInspection = &inspection
} else {
response.Type = protocol.ResourceReconciled
if request.ResourceRecovery == nil {
err = fmt.Errorf("resource recovery evidence is required")
} else {
err = stores[0].ReconcileResource(ctx, *request.ResourceRecovery)
}
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence) {
var err error
switch request.Type {
@@ -365,19 +325,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
}
mux.HandleFunc("GET /connect", connect)
mux.HandleFunc("GET /operator-connect", connect)
return &runHandler{Handler: mux, runner: runs}
}
func isLoopbackPeer(remoteAddr string) bool {
host, _, err := net.SplitHostPort(remoteAddr)
if err != nil {
return false
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
type runHandler struct {
http.Handler
runner *runner