Remove resource pools, leases and wait queues
Drop the resource allocation feature end to end: store tables and migration, runner scheduling and cleanup, protocol types and commands, CLI commands, TUI screens, tests and docs. Existing databases open cleanly and definitions carrying a retired resources block still validate.
This commit is contained in:
@@ -1,511 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/client"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
"github.com/coder/websocket"
|
||||
"github.com/coder/websocket/wsjson"
|
||||
)
|
||||
|
||||
type fixedRemoteListener struct{ net.Listener }
|
||||
|
||||
func (l fixedRemoteListener) Accept() (net.Conn, error) {
|
||||
conn, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fixedRemoteConn{Conn: conn}, nil
|
||||
}
|
||||
|
||||
type fixedRemoteConn struct{ net.Conn }
|
||||
|
||||
func (fixedRemoteConn) RemoteAddr() net.Addr {
|
||||
return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443}
|
||||
}
|
||||
|
||||
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
|
||||
// executor. It never calls inference, a phone, or a deployed server.
|
||||
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
|
||||
t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json"))
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n"
|
||||
if _, err := store.Submit(ctx, "scratch-resource", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store).(*runHandler)
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if input["prompt"] == "A" {
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil
|
||||
}
|
||||
handler.runner.events.interval = time.Millisecond
|
||||
events := make(chan protocol.RunEventFrame, 64)
|
||||
sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error {
|
||||
if m.RunEvent != nil {
|
||||
events <- *m.RunEvent
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sub.ready()
|
||||
defer sub.close()
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
run := func(args ...string) string {
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
|
||||
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
|
||||
}
|
||||
t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String())
|
||||
return out.String()
|
||||
}
|
||||
parseRun := func(output string) int64 {
|
||||
var m protocol.Message
|
||||
if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil {
|
||||
t.Fatal("CLI run response", output, err)
|
||||
}
|
||||
return m.RunDetail.ID
|
||||
}
|
||||
run("validate", "testdata/android-resource-draft.md", "--json")
|
||||
a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a"))
|
||||
<-started
|
||||
b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b"))
|
||||
awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
var detail protocol.RunDetail
|
||||
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 {
|
||||
t.Fatal("CLI did not observe queue", err)
|
||||
}
|
||||
if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") {
|
||||
t.Fatal("list omitted resource status", output)
|
||||
}
|
||||
var snapshot *protocol.RunEventFrame
|
||||
handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error {
|
||||
if m.RunEvent != nil {
|
||||
snapshot = m.RunEvent
|
||||
}
|
||||
return nil
|
||||
})
|
||||
handler.runner.events.unsubscribeClient("snapshot-evidence")
|
||||
if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet {
|
||||
t.Fatal("subscription snapshot omitted resource wait")
|
||||
}
|
||||
close(release)
|
||||
run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json")
|
||||
detail = protocol.RunDetail{}
|
||||
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 {
|
||||
t.Fatal("automatic progress not observed", err)
|
||||
}
|
||||
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
|
||||
}
|
||||
|
||||
func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"
|
||||
if _, err := store.Submit(ctx, "operator-resource", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
|
||||
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
|
||||
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
|
||||
}
|
||||
if err := store.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
|
||||
t.Fatalf("ordinary connect accepted recovery: %v", err)
|
||||
}
|
||||
normal.Close()
|
||||
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
runCLI := func(args ...string) string {
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
|
||||
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
inspection := runCLI("resource-inspect", visit.Resource, "--json")
|
||||
if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") {
|
||||
t.Fatalf("inspection leaked capability: %s", inspection)
|
||||
}
|
||||
runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json")
|
||||
inspection = runCLI("resource-inspect", visit.Resource, "--json")
|
||||
var recovered protocol.ResourceInspection
|
||||
if err := json.Unmarshal([]byte(inspection), &recovered); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" {
|
||||
t.Fatalf("CLI recovery state was not persisted: %s", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
|
||||
lease, ch, _, err := store.RequestResource(ctx, visit)
|
||||
if err != nil || ch != nil {
|
||||
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
|
||||
}
|
||||
if err := store.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
runCLI := func(args ...string) error {
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
err := app.Run(ctx, append([]string{"--url", endpoint}, args...))
|
||||
if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") {
|
||||
t.Fatalf("private capability appeared in CLI output: %s", out.String())
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
|
||||
t.Fatal("accepted wrong owner")
|
||||
}
|
||||
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
|
||||
t.Fatal("accepted stale stage")
|
||||
}
|
||||
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil {
|
||||
t.Fatal("accepted stale sequence")
|
||||
}
|
||||
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
|
||||
t.Fatal("accepted non-quarantined owner")
|
||||
}
|
||||
inspection, err := store.InspectResource(ctx, visit.Resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 {
|
||||
t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) {
|
||||
for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} {
|
||||
if isLoopbackPeer(remote) {
|
||||
t.Fatalf("accepted non-loopback peer %q", remote)
|
||||
}
|
||||
}
|
||||
for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} {
|
||||
if !isLoopbackPeer(remote) {
|
||||
t.Fatalf("rejected loopback peer %q", remote)
|
||||
}
|
||||
}
|
||||
// X-Forwarded-For is deliberately not consulted: a proxy-supplied header
|
||||
// cannot turn a remote WebSocket peer into the local operator transport.
|
||||
}
|
||||
|
||||
func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
|
||||
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
|
||||
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
|
||||
}
|
||||
if err := store.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := store.InspectResource(ctx, visit.Resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
server := httptest.NewUnstartedServer(Handler(ctx, store))
|
||||
server.Listener = fixedRemoteListener{Listener: server.Listener}
|
||||
server.Start()
|
||||
defer server.Close()
|
||||
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
var welcome protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var refused protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &refused); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") {
|
||||
t.Fatalf("forged peer was accepted: %#v", refused)
|
||||
}
|
||||
after, err := store.InspectResource(ctx, visit.Resource)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(before, after) {
|
||||
t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after)
|
||||
}
|
||||
|
||||
trusted := httptest.NewServer(Handler(ctx, store))
|
||||
defer trusted.Close()
|
||||
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
|
||||
t.Fatalf("ordinary connect accepted recovery: %v", err)
|
||||
}
|
||||
normal.Close()
|
||||
operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer operator.Close()
|
||||
if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil {
|
||||
t.Fatalf("trusted operator transport refused inspection: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolOperatorCLIConfigureReadbackAndPreserveLease(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
dbPath := filepath.Join(t.TempDir(), "pool-operator.db")
|
||||
store, err := pipeline.Open(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if err := store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "old"}}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Submit(ctx, "pool-owner", pipeline.Template([]string{"work"}, true)+"```resources\n{\"pool\":\"targets\"}\n```\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil || len(items) != 1 {
|
||||
t.Fatal("pool pipeline missing", err)
|
||||
}
|
||||
detail, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runID, _, err := store.CreateRun(ctx, detail, "pool-owner", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, ready, _, err := store.RequestResourcePool(ctx, "targets", pipeline.ResourceVisit{Resource: "targets", RunID: runID, Stage: "work", Sequence: 1})
|
||||
if err != nil || ready != nil {
|
||||
t.Fatalf("pool lease: ready=%v err=%v", ready, err)
|
||||
}
|
||||
|
||||
handler := Handler(ctx, store)
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
normal, err := client.Open(ctx, endpoint)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.ConfigureResourcePool, ResourcePool: &protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true}}}}); err == nil || !strings.Contains(err.Error(), "operator transport") {
|
||||
t.Fatalf("remote configuration was accepted: %v", err)
|
||||
}
|
||||
normal.Close()
|
||||
|
||||
declaration := filepath.Join(t.TempDir(), "pool.json")
|
||||
if err := os.WriteFile(declaration, []byte(`{"name":"targets","members":[{"member":"phone","enabled":true,"metadata":{"serial":"new"}},{"member":"emulator","enabled":false}]}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
for _, fromStdin := range []bool{false, true} {
|
||||
oversized := strings.Repeat(" ", 256*1024+1)
|
||||
path := filepath.Join(t.TempDir(), "oversized.json")
|
||||
if err := os.WriteFile(path, []byte(oversized), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fromStdin {
|
||||
path = "-"
|
||||
}
|
||||
bounded := cli.App{Out: &out, Err: &out, In: strings.NewReader(oversized)}
|
||||
err := bounded.Run(ctx, []string{"--url", endpoint, "resource-pool-configure", path, "--yes", "--json"})
|
||||
if err == nil || !strings.Contains(err.Error(), "exceeds 262144 bytes") {
|
||||
t.Fatalf("oversized pool input (stdin=%v): %v", fromStdin, err)
|
||||
}
|
||||
out.Reset()
|
||||
}
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http"), "resource-pool-configure", declaration, "--yes", "--json"}); err != nil {
|
||||
t.Fatalf("configure CLI: %v\n%s", err, out.String())
|
||||
}
|
||||
out.Reset()
|
||||
app = cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http"), "resource-pool-inspect", "targets", "--json"}); err != nil {
|
||||
t.Fatalf("inspect CLI: %v\n%s", err, out.String())
|
||||
}
|
||||
var pool protocol.ResourcePool
|
||||
if err := json.Unmarshal(out.Bytes(), &pool); err != nil || len(pool.Members) != 2 || pool.Members[0].Metadata["serial"] != "new" {
|
||||
t.Fatalf("bad pool readback: %s (%v)", out.String(), err)
|
||||
}
|
||||
inspection, err := store.InspectResource(ctx, "phone")
|
||||
if err != nil || inspection.Owner == nil || inspection.Owner.RunID != runID {
|
||||
t.Fatalf("configuration released active lease: %#v (%v)", inspection, err)
|
||||
}
|
||||
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.ReleaseResource(ctx, lease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.Close()
|
||||
store.Close()
|
||||
reopened, err := pipeline.Open(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
persisted, err := reopened.InspectResourcePool(ctx, "targets")
|
||||
if err != nil || persisted.Members[0].Metadata["serial"] != "new" {
|
||||
t.Fatalf("pool did not survive reopen: %#v (%v)", persisted, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
|
||||
baseline, err := os.ReadFile("testdata/android-v3-published.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" {
|
||||
t.Fatal("baseline is not retrieved Android v3", hash)
|
||||
}
|
||||
draft, err := os.ReadFile("testdata/android-resource-draft.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, err := pipeline.Parse(string(baseline))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
updated, err := pipeline.Parse(string(draft))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(old.Stages) != len(updated.Stages) {
|
||||
t.Fatal("draft changed graph shape")
|
||||
}
|
||||
for i := range old.Stages {
|
||||
a, b := old.Stages[i], updated.Stages[i]
|
||||
if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] {
|
||||
t.Fatal("draft changed approved model")
|
||||
}
|
||||
if b.Key == "ui_reviewer" {
|
||||
if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" {
|
||||
t.Fatal("reviewer resource missing")
|
||||
}
|
||||
} else if b.Resources != nil {
|
||||
t.Fatal("non-UI stage acquired phone")
|
||||
}
|
||||
a.Prompt, b.Prompt = "", ""
|
||||
a.Resources, b.Resources = nil, nil
|
||||
if !reflect.DeepEqual(a, b) {
|
||||
t.Fatal("draft changed fields outside approved role split/resource", a.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func (r *runner) acquireResource(ctx context.Context, visit pipeline.ResourceVisit, workspace string, completed int) (pipeline.ResourceLease, error) {
|
||||
return r.acquireResourceWithPool(ctx, visit, "", workspace, completed)
|
||||
}
|
||||
|
||||
func (r *runner) acquireResourceWithPool(ctx context.Context, visit pipeline.ResourceVisit, pool string, workspace string, completed int) (pipeline.ResourceLease, error) {
|
||||
var lease pipeline.ResourceLease
|
||||
var ready <-chan pipeline.ResourceLease
|
||||
var wait *protocol.ResourceWait
|
||||
var err error
|
||||
if pool != "" {
|
||||
lease, ready, wait, err = r.store.RequestResourcePool(ctx, pool, visit)
|
||||
} else {
|
||||
lease, ready, wait, err = r.store.RequestResource(ctx, visit)
|
||||
}
|
||||
if err != nil {
|
||||
return lease, err
|
||||
}
|
||||
if ready != nil {
|
||||
r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, Status: protocol.RunStatusQueued, CurrentStage: visit.Stage, CompletedSteps: completed, ResourceWaitSet: true, ResourceWait: wait}, false)
|
||||
select {
|
||||
case lease = <-ready:
|
||||
r.store.AcceptResource(lease)
|
||||
case <-ctx.Done():
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := r.store.CancelResource(cleanup, visit); err != nil {
|
||||
return lease, errors.Join(ctx.Err(), err)
|
||||
}
|
||||
return lease, ctx.Err()
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
err := r.releaseResource(lease, true) // Never launched, including a concurrent grant.
|
||||
return pipeline.ResourceLease{}, errors.Join(ctx.Err(), err)
|
||||
}
|
||||
r.emitRun(protocol.RunEventFrame{RunID: visit.RunID, Workspace: workspace, ResourceWaitSet: true}, false)
|
||||
return lease, nil
|
||||
}
|
||||
|
||||
func (r *runner) releaseResource(lease pipeline.ResourceLease, confirmed bool) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := r.store.SetResourceCleanup(ctx, lease, confirmed); err != nil {
|
||||
return err
|
||||
}
|
||||
if confirmed {
|
||||
return r.store.ReleaseResource(ctx, lease)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *runner) resourceQuiescent(id int64, executionErr error, panicked bool) bool {
|
||||
if panicked || errors.Is(executionErr, stage.ErrStageCleanup) {
|
||||
return false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
sessions, err := r.store.ActiveAgentSessionsForRun(ctx, id)
|
||||
return err == nil && len(sessions) == 0
|
||||
}
|
||||
@@ -1,418 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func resourceRunner(t *testing.T) (*runner, <-chan protocol.RunEventFrame, func(string, string, string) *protocol.RunDetail) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "resources.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := newRunner(ctx, s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.events = newRunEventHub()
|
||||
r.events.interval = time.Millisecond
|
||||
events := make(chan protocol.RunEventFrame, 128)
|
||||
sub := r.events.subscribe("resource-test", "", nil, func(m protocol.Message) error {
|
||||
if m.RunEvent != nil {
|
||||
events <- *m.RunEvent
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sub.ready()
|
||||
t.Cleanup(func() { cancel(); r.wg.Wait(); sub.close(); s.Close() })
|
||||
count := 0
|
||||
return r, events, func(resource, timeout, prompt string) *protocol.RunDetail {
|
||||
count++
|
||||
source := strings.Replace(pipeline.Template([]string{"work"}, true), "timeout=1h", "timeout="+timeout, 1)
|
||||
if resource != "" {
|
||||
source += "```resources\n{\"exclusive\":\"" + resource + "\"}\n```\n"
|
||||
}
|
||||
name := fmt.Sprint("resource-", count)
|
||||
if _, err := s.Submit(ctx, name, source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
var pid int64
|
||||
for _, p := range items {
|
||||
if p.Name == name {
|
||||
pid = p.ID
|
||||
}
|
||||
}
|
||||
d, err := r.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: name, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
}
|
||||
|
||||
func awaitResourceEvent(t *testing.T, events <-chan protocol.RunEventFrame, id int64, predicate func(protocol.RunEventFrame) bool) protocol.RunEventFrame {
|
||||
t.Helper()
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case event := <-events:
|
||||
if event.RunID == id && predicate(event) {
|
||||
return event
|
||||
}
|
||||
case <-timer.C:
|
||||
t.Fatalf("resource event missing for run %d", id)
|
||||
return protocol.RunEventFrame{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePauseBeyondTimeoutAndAutomaticResume(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
aStarted, releaseA := make(chan struct{}), make(chan struct{})
|
||||
bStarted := make(chan struct{}, 1)
|
||||
var bCalls atomic.Int32
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
switch input["prompt"] {
|
||||
case "A":
|
||||
close(aStarted)
|
||||
select {
|
||||
case <-releaseA:
|
||||
case <-ctx.Done():
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
case "B":
|
||||
bCalls.Add(1)
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || time.Until(deadline) < 15*time.Millisecond {
|
||||
return stage.Result{}, errors.New("execution timeout was consumed in queue")
|
||||
}
|
||||
bStarted <- struct{}{}
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "2s", "A")
|
||||
<-aStarted
|
||||
b := start("X", "30ms", "B")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
<-time.After(70 * time.Millisecond) // Deliberately longer than B's whole execution timeout.
|
||||
d, err := r.store.GetRun(context.Background(), b.ID)
|
||||
if err != nil || d.Status != protocol.RunStatusQueued || d.ResourceWait == nil || len(d.Steps) != 0 || bCalls.Load() != 0 {
|
||||
t.Fatal("queued execution consumed accounting", d, err, bCalls.Load())
|
||||
}
|
||||
if _, err := r.decide(protocol.Message{RunID: b.ID, Option: "success"}); err == nil {
|
||||
t.Fatal("resource wait exposed a human decision")
|
||||
}
|
||||
c := start("Y", "1s", "C")
|
||||
awaitResourceEvent(t, events, c.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
select {
|
||||
case <-bStarted:
|
||||
t.Fatal("B ran while A held X")
|
||||
default:
|
||||
}
|
||||
owner, err := r.store.GetRun(context.Background(), a.ID)
|
||||
if err != nil || owner.Status != "running" {
|
||||
t.Fatal("A did not remain owner", err)
|
||||
}
|
||||
close(releaseA)
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
r.wg.Wait()
|
||||
d, err = r.store.GetRun(context.Background(), b.ID)
|
||||
if err != nil || bCalls.Load() != 1 || d.ResourceWait != nil || len(d.Steps) != 1 || d.Steps[0].Control.Visit != 1 || d.Steps[0].Sequence != 1 {
|
||||
t.Fatal("resume did not execute exactly one fresh visit", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcePoolRunnerHandoffUsesConcreteTargetAfterQueue(t *testing.T) {
|
||||
r, events, _ := resourceRunner(t)
|
||||
ctx := context.Background()
|
||||
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"pool\":\"targets\"}\n```\n"
|
||||
if _, err := r.store.Submit(ctx, "pool-runner", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := r.store.List(ctx)
|
||||
if err != nil || len(items) != 1 {
|
||||
t.Fatal("pool definition missing", err)
|
||||
}
|
||||
definition, err := r.store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{
|
||||
{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "phone-serial", "type": "phone"}},
|
||||
{Member: "emulator", Enabled: true, Metadata: map[string]string{"serial": "emu-serial", "type": "emulator"}},
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ownerRun, _, err := r.store.CreateRun(ctx, definition, "phone-owner", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
phone, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "phone", RunID: ownerRun, Stage: "work", Sequence: 1})
|
||||
if err != nil || ch != nil {
|
||||
t.Fatalf("phone owner acquire: channel=%v err=%v", ch, err)
|
||||
}
|
||||
defer r.releaseResource(phone, true)
|
||||
|
||||
aStarted, releaseA := make(chan struct{}), make(chan struct{})
|
||||
bRan := make(chan *protocol.ResourceTarget, 1)
|
||||
r.execute = func(stageCtx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
target, ok := stage.ResourceTarget(stageCtx)
|
||||
if !ok || target == nil {
|
||||
return stage.Result{}, errors.New("missing resource target context")
|
||||
}
|
||||
if input["prompt"] == "A" {
|
||||
if target.Member != "emulator" || target.Metadata["serial"] != "emu-serial" || target.Metadata["type"] != "emulator" {
|
||||
return stage.Result{}, fmt.Errorf("wrong first target: %#v", target)
|
||||
}
|
||||
close(aStarted)
|
||||
<-releaseA
|
||||
} else {
|
||||
if deadline, ok := stageCtx.Deadline(); !ok || time.Until(deadline) < 15*time.Millisecond {
|
||||
return stage.Result{}, errors.New("execution deadline was consumed in pool queue")
|
||||
}
|
||||
bRan <- target
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
start := func(key, prompt string) *protocol.RunDetail {
|
||||
run, err := r.start(protocol.Message{PipelineID: definition.ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": prompt}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return run
|
||||
}
|
||||
a := start("pool-a", "A")
|
||||
<-aStarted
|
||||
b := start("pool-b", "B")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
select {
|
||||
case <-bRan:
|
||||
t.Fatal("pool waiter ran before either member was released")
|
||||
default:
|
||||
}
|
||||
if err := r.releaseResource(phone, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var target *protocol.ResourceTarget
|
||||
select {
|
||||
case target = <-bRan:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("pool waiter did not resume")
|
||||
}
|
||||
if target.Member != "phone" || target.Metadata["serial"] != "phone-serial" {
|
||||
t.Fatalf("wrong queued target: %#v", target)
|
||||
}
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
close(releaseA)
|
||||
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
r.wg.Wait()
|
||||
detail, err := r.store.GetRun(ctx, b.ID)
|
||||
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Control.ResourceTarget == nil {
|
||||
t.Fatalf("pool target evidence missing: detail=%#v err=%v", detail, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceCancellationWaitsForOwnerCleanup(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
started, cleaning, cleanupDone := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
var waitingCalls atomic.Int32
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if input["prompt"] == "owner" {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
close(cleaning)
|
||||
<-cleanupDone
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
waitingCalls.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "2s", "owner")
|
||||
<-started
|
||||
b := start("X", "30ms", "waiter")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
if err := r.cancel(a.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-cleaning
|
||||
select {
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
if waitingCalls.Load() != 0 {
|
||||
t.Fatal("handoff before cleanup")
|
||||
}
|
||||
}
|
||||
close(cleanupDone)
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
if waitingCalls.Load() != 1 {
|
||||
t.Fatal("waiter did not resume exactly once")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceQuarantineAndQueuedCancellation(t *testing.T) {
|
||||
for _, failure := range []string{"cleanup", "panic", "restoration"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
r.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if input["prompt"] == "owner" {
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
if failure == "panic" {
|
||||
panic("unknown cleanup")
|
||||
}
|
||||
if failure == "restoration" {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{protocol.ResourceCleanupConfirmedKey: false}}, nil
|
||||
}
|
||||
return stage.Result{}, stage.ErrStageCleanup
|
||||
}
|
||||
calls.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "1s", "owner")
|
||||
<-started
|
||||
b := start("X", "30ms", "waiter")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
|
||||
close(release)
|
||||
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
||||
if err := r.cancel(b.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool {
|
||||
return e.Status == "canceled" && e.ResourceWaitSet && e.ResourceWait == nil
|
||||
})
|
||||
r.wg.Wait()
|
||||
if calls.Load() != 0 {
|
||||
t.Fatal("quarantined resource handed off")
|
||||
}
|
||||
exists, err := r.store.HasResourceOwner(context.Background(), a.ID)
|
||||
if err != nil || !exists {
|
||||
t.Fatal("quarantine evidence lost", err)
|
||||
}
|
||||
d, err := r.store.GetRun(context.Background(), b.ID)
|
||||
if err != nil || d.ResourceWait != nil || len(d.Steps) != 0 {
|
||||
t.Fatal("canceled queue consumed step", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceEventClearCoalescing(t *testing.T) {
|
||||
frame := protocol.RunEventFrame{ResourceWaitSet: true, ResourceWait: &protocol.ResourceWait{Resource: "X"}}
|
||||
mergeRunEventFrame(&frame, &protocol.RunEventFrame{ResourceWaitSet: true})
|
||||
if !frame.ResourceWaitSet || frame.ResourceWait != nil {
|
||||
t.Fatal("coalescer retained stale resource wait")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceReleaseOnBeginAndPersistenceFailure(t *testing.T) {
|
||||
for _, failure := range []string{"begin", "persist"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
db, err := sql.Open("sqlite", filepath.Join(r.store.Directory(), "resources.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
trigger := `CREATE TRIGGER resource_fail BEFORE INSERT ON run_steps WHEN json_extract(NEW.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test BeginStep failure'); END`
|
||||
if failure == "persist" {
|
||||
trigger = `CREATE TRIGGER resource_fail BEFORE UPDATE OF output_json ON run_steps WHEN json_extract(OLD.input_json,'$.prompt')='fail' BEGIN SELECT RAISE(FAIL,'test persistence failure'); END`
|
||||
}
|
||||
if _, err := db.Exec(trigger); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls atomic.Int32
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
calls.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "1s", "fail")
|
||||
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
||||
r.wg.Wait()
|
||||
if exists, err := r.store.HasResourceOwner(context.Background(), a.ID); err != nil || exists {
|
||||
t.Fatal("safe failure stranded resource", err)
|
||||
}
|
||||
if failure == "begin" && calls.Load() != 0 {
|
||||
t.Fatal("BeginStep failure executed work")
|
||||
}
|
||||
b := start("X", "1s", "pass")
|
||||
awaitResourceEvent(t, events, b.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceCleanupQueryFailureQuarantines(t *testing.T) {
|
||||
r, events, start := resourceRunner(t)
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
if err := r.store.Close(); err != nil {
|
||||
return stage.Result{}, err
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
a := start("X", "1s", "query-failure")
|
||||
awaitResourceEvent(t, events, a.ID, func(e protocol.RunEventFrame) bool { return e.Status == "failed" })
|
||||
r.wg.Wait()
|
||||
reopened, err := pipeline.Open(filepath.Join(r.store.Directory(), "resources.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
if err := reopened.RecoverRuns(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exists, err := reopened.HasResourceOwner(context.Background(), a.ID); err != nil || !exists {
|
||||
t.Fatal("query failure released uncertain owner", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceSkippedVisitDoesNotAcquire(t *testing.T) {
|
||||
r, events, _ := resourceRunner(t)
|
||||
ctx := context.Background()
|
||||
source := strings.Replace(pipeline.Template([]string{"review"}, true), "success -> #success", "success -> #success\nskipped -> #success", 1) + "```resources\n{\"exclusive\":\"X\"}\n```\n"
|
||||
if _, err := r.store.Submit(ctx, "skip", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := r.store.List(ctx)
|
||||
d, _ := r.store.Get(ctx, items[0].ID, 1)
|
||||
owner, _, err := r.store.CreateRun(ctx, d, "held", t.TempDir(), false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, ch, _, err := r.store.RequestResource(ctx, pipeline.ResourceVisit{Resource: "X", RunID: owner, Stage: "review", Sequence: 1})
|
||||
if err != nil || ch != nil {
|
||||
t.Fatal("owner fixture failed", err)
|
||||
}
|
||||
defer r.releaseResource(lease, true)
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
t.Error("skipped stage executed")
|
||||
return stage.Result{}, nil
|
||||
}
|
||||
run, err := r.start(protocol.Message{PipelineID: d.ID, Version: 1, RequestKey: "skip", Workspace: t.TempDir(), Input: map[string]any{"prompt": "no UI"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
awaitResourceEvent(t, events, run.ID, func(e protocol.RunEventFrame) bool { return e.Status == "succeeded" })
|
||||
detail, err := r.store.GetRun(ctx, run.ID)
|
||||
if err != nil || len(detail.Steps) != 1 || detail.Steps[0].Outcome != "skipped" {
|
||||
t.Fatal("skip not preserved", detail, err)
|
||||
}
|
||||
}
|
||||
@@ -424,9 +424,6 @@ func (s *runSubscription) close() {
|
||||
}
|
||||
|
||||
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
||||
if in.ResourceWaitSet {
|
||||
into.ResourceWaitSet, into.ResourceWait = true, in.ResourceWait
|
||||
}
|
||||
if in.Workspace != "" {
|
||||
into.Workspace = in.Workspace
|
||||
}
|
||||
@@ -503,20 +500,18 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
continue
|
||||
}
|
||||
frame := protocol.RunEventFrame{
|
||||
ResourceWaitSet: true,
|
||||
ResourceWait: summary.ResourceWait,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
}
|
||||
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
|
||||
// The connection write failed; the subscription is closed and the
|
||||
|
||||
+15
-83
@@ -86,10 +86,6 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
}
|
||||
cancel()
|
||||
if err != nil {
|
||||
if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource {
|
||||
log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID)
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
||||
}
|
||||
if err := s.ConfirmRecoveredSession(ctx, owner); err != nil {
|
||||
@@ -224,9 +220,6 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
|
||||
// workspace) must not block new work.
|
||||
for _, owner := range owners {
|
||||
if _, ok := r.active[owner.RunID]; !ok {
|
||||
if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace {
|
||||
continue
|
||||
}
|
||||
r.mu.Unlock()
|
||||
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
||||
}
|
||||
@@ -452,7 +445,7 @@ func retryInputOverride(base map[string]any, override map[string]any) (map[strin
|
||||
// retryStageContractEqual compares the complete persisted stage contract. A
|
||||
// retry may carry completed output across a published version only when the
|
||||
// stages that produced that output are unchanged; comparing only effective
|
||||
// types would allow prompt, config, edge, resource, or limit changes through.
|
||||
// types would allow prompt, config, edge, or limit changes through.
|
||||
func retryStageContractEqual(left, right protocol.Stage) bool {
|
||||
a, err := json.Marshal(left)
|
||||
if err != nil {
|
||||
@@ -1182,14 +1175,13 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
|
||||
// Push the terminal delta before the attention notification, so a slow ntfy
|
||||
// never delays the live update the app folds.
|
||||
r.emitRun(protocol.RunEventFrame{
|
||||
RunID: id,
|
||||
Workspace: workspace,
|
||||
Status: status,
|
||||
ResourceWaitSet: true,
|
||||
Outcome: outcome,
|
||||
CurrentStage: currentStage,
|
||||
CompletedSteps: completed,
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
RunID: id,
|
||||
Workspace: workspace,
|
||||
Status: status,
|
||||
Outcome: outcome,
|
||||
CurrentStage: currentStage,
|
||||
CompletedSteps: completed,
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}, true)
|
||||
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
|
||||
// The run is terminal: nothing more can notify for it.
|
||||
@@ -1257,21 +1249,6 @@ func (r *runner) finishStage(id int64, name, workspace string, selected protocol
|
||||
}
|
||||
|
||||
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
|
||||
var held *pipeline.ResourceLease
|
||||
resourceQuiet := true
|
||||
release := func() error {
|
||||
if held != nil {
|
||||
err := r.releaseResource(*held, resourceQuiet)
|
||||
held = nil
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if err := release(); err != nil {
|
||||
log.Printf("resource cleanup run=%d: %v", id, err)
|
||||
}
|
||||
}()
|
||||
finish := func(status, outcome, diagnostic string) {
|
||||
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
|
||||
}
|
||||
@@ -1336,7 +1313,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
maxVisits = selected.Limits.MaxVisits
|
||||
}
|
||||
// Validate the next visit without consuming it. An already-exhausted
|
||||
// stage must not queue for a resource it can never execute.
|
||||
// stage must not begin work it can never execute.
|
||||
if !isHuman {
|
||||
if state.visits[key]+1 > 5 {
|
||||
finish("failed", "error", "stage visit budget exceeded")
|
||||
@@ -1349,27 +1326,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
_, declaresSkip := def.Edges["skipped"]
|
||||
skip := !isHuman && declaresSkip && !state.uiSurface
|
||||
if selected.Resources != nil && !skip {
|
||||
if isHuman || (selected.Resources.Exclusive == "" && selected.Resources.Pool == "") || (selected.Resources.Exclusive != "" && selected.Resources.Pool != "") || (selected.Resources.Exclusive != "" && !pipeline.ValidResourceKey(selected.Resources.Exclusive)) || (selected.Resources.Pool != "" && !pipeline.ValidResourceKey(selected.Resources.Pool)) {
|
||||
finish("failed", "error", "invalid persisted resource declaration")
|
||||
return
|
||||
}
|
||||
resource := selected.Resources.Exclusive
|
||||
pool := selected.Resources.Pool
|
||||
if pool != "" {
|
||||
resource = pool
|
||||
}
|
||||
lease, err := r.acquireResourceWithPool(ctx, pipeline.ResourceVisit{Resource: resource, RunID: id, Stage: key, Sequence: sequence}, pool, workspace, state.completed)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", err.Error())
|
||||
} else {
|
||||
finish("failed", "error", err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
held, resourceQuiet = &lease, true
|
||||
}
|
||||
state.nextSequence = sequence + 1
|
||||
state.visits[key]++
|
||||
// Publish the run's server-captured changed paths into the input memory
|
||||
@@ -1403,13 +1359,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
// queued -> running: report the new current stage and the step start.
|
||||
r.emitRun(protocol.RunEventFrame{
|
||||
RunID: id,
|
||||
Workspace: workspace,
|
||||
Status: protocol.RunStatusRunning,
|
||||
CurrentStage: key,
|
||||
ResourceWaitSet: true,
|
||||
CompletedSteps: state.completed,
|
||||
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
|
||||
RunID: id,
|
||||
Workspace: workspace,
|
||||
Status: protocol.RunStatusRunning,
|
||||
CurrentStage: key,
|
||||
CompletedSteps: state.completed,
|
||||
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
|
||||
}, false)
|
||||
if isHuman {
|
||||
// A human step is O(1): one BeginStep plus one MarkWaiting, no
|
||||
@@ -1516,9 +1471,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
var piClient *stage.PiClient
|
||||
var err error
|
||||
stageCtx := ctx
|
||||
if held != nil && held.Target != nil {
|
||||
stageCtx = stage.WithResourceTarget(stageCtx, held.Target)
|
||||
}
|
||||
var modelConfig stage.ModelConfig
|
||||
var budgetBefore stage.BudgetSnapshot
|
||||
if stage.EffectiveType(def.Type) == stage.ModelType {
|
||||
@@ -1542,9 +1494,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
snapshotCancel()
|
||||
var stageSession string
|
||||
panicked := false
|
||||
if held != nil {
|
||||
resourceQuiet = false
|
||||
}
|
||||
func() {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
@@ -1600,12 +1549,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
|
||||
}
|
||||
}()
|
||||
if held != nil {
|
||||
resourceQuiet = r.resourceQuiescent(id, err, panicked)
|
||||
if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true {
|
||||
resourceQuiet = false
|
||||
}
|
||||
}
|
||||
// Capture after cleanup, even when the stage's context was canceled.
|
||||
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
after := filediff.Capture(snapshotCtx, workspace, before)
|
||||
@@ -1698,9 +1641,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
step.Target = def.Edges[step.Outcome]
|
||||
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
|
||||
if held != nil && held.Target != nil {
|
||||
step.Control.ResourceTarget = held.Target
|
||||
}
|
||||
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
|
||||
for _, outcome := range selected.Limits.On {
|
||||
if outcome == step.Outcome {
|
||||
@@ -1765,14 +1705,6 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
return
|
||||
}
|
||||
}
|
||||
if held != nil && !resourceQuiet {
|
||||
finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection")
|
||||
return
|
||||
}
|
||||
if err := release(); err != nil {
|
||||
finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error())
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
finish("canceled", "", ctx.Err().Error())
|
||||
return
|
||||
|
||||
-312
@@ -1,312 +0,0 @@
|
||||
# stage implementer
|
||||
|
||||
```limits
|
||||
{"max_visits":5,"on":["blocked_retryable"]}
|
||||
```
|
||||
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "code_findings": "reviewer.findings?", "ui_findings": "ui_reviewer.findings?"}
|
||||
```
|
||||
|
||||
```output
|
||||
{ task, acceptance_checks, changed_files, implementation_summary, new_helpers }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Implement input.prompt in the run directory using the repository's actual languages and
|
||||
file types; create files as needed.
|
||||
|
||||
Read android/AGENTS.md first. It is the map of this project - how to build, test, run and capture here, where
|
||||
the pieces live, and the conventions that apply. Do not re-derive what it tells you and do not go
|
||||
looking for what it already names; if it is wrong or out of date, say so in your output.
|
||||
|
||||
Read the surrounding code before editing so you extend the existing helpers and patterns rather than
|
||||
duplicating them. Keep each change as small as the task allows and follow the repository's own
|
||||
conventions and error handling.
|
||||
|
||||
This is the Android app. Build and test it with the project's own Gradle invocation as the map
|
||||
gives it, and treat the map's geometry tokens as the only source of dimensions - a UI change that
|
||||
adds a raw value fails the repository's own check. Perform code/build/unit tests only;
|
||||
no adb, install, capture, device-setting changes or restoring main's build. The UI reviewer
|
||||
alone verifies rendered surfaces under its declared runner resource lease.
|
||||
|
||||
Record the acceptance checks in acceptance_checks, the actual changes in changed_files, and any new
|
||||
shared helper in new_helpers. Verify with the checks this repository uses - as the map gives them -
|
||||
and report the exact commands and observed output. Address any concrete findings the runner supplies
|
||||
and note the actual fixes. Return blocked if the required scope or tools are unavailable. Do not
|
||||
commit or push; the committer stage lands the work.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
edit
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-terra
|
||||
retries=1
|
||||
timeout=30m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
success -> reviewer
|
||||
blocked_retryable -> implementer
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
|
||||
# stage reviewer
|
||||
|
||||
```limits
|
||||
{"max_visits": 3, "on": ["rejected"]}
|
||||
```
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
|
||||
```
|
||||
|
||||
|
||||
```output
|
||||
{ summary, changed_files, verification_evidence, duplication_review, performance_review, findings }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Verify with builds, unit tests and source review as android/AGENTS.md gives them.
|
||||
Do not use adb, install, capture, change device settings or restore main's build.
|
||||
Device verification belongs exclusively to the UI reviewer under its runner resource lease.
|
||||
Enforce the map's geometry-token rule: a raw dimension outside the token file is a must-fix finding.
|
||||
|
||||
Review the actual changes against the task in input.prompt and the acceptance
|
||||
checks the implementer recorded. Run the repository's relevant verification checks
|
||||
with the stage tools (for example build, lint, tests or type checks) and report the
|
||||
exact commands and observed results in verification_evidence.
|
||||
|
||||
Independently assess code quality:
|
||||
- Duplication: search the run directory for repeated or near-identical functions,
|
||||
blocks and copy-pasted logic introduced by the change; identify code that should be
|
||||
shared and record concrete locations in duplication_review.
|
||||
- Performance and resources: flag redundant computation, unnecessary dependencies,
|
||||
and repeated I/O or database/filesystem work that wastes time, CPU or memory;
|
||||
record it in performance_review.
|
||||
- Workarounds: flag hacks, suppressed errors and shortcuts that trade reliability for
|
||||
speed in findings.
|
||||
|
||||
Keep verification proportionate to the change: exercise the files and screens the diff
|
||||
actually touches, capture at most one screenshot per changed surface plus one regression
|
||||
screen, and do not repeat sweeps or measurements that an earlier visit already recorded.
|
||||
|
||||
Return rejected ONLY when at least one must-fix finding exists: a defect that blocks
|
||||
acceptance and that you can locate concretely (file and line), with evidence and a
|
||||
remedy. Do not return rejected for advisory-only, cosmetic or nit items. When there
|
||||
is no must-fix finding but you have non-blocking notes (duplication, performance or
|
||||
workaround observations), record them in findings and return advisory, which
|
||||
continues the run without another implementer visit. Return success only when the
|
||||
observed checks pass and you have nothing further to record. Do not edit files or
|
||||
claim unexecuted checks.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-luna
|
||||
retries=1
|
||||
timeout=45m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
success -> ui_reviewer
|
||||
advisory -> ui_reviewer
|
||||
rejected -> implementer
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
|
||||
# stage ui_reviewer
|
||||
|
||||
```resources
|
||||
{"exclusive":"phone-47241FDAS00DLK"}
|
||||
```
|
||||
|
||||
```limits
|
||||
{"max_visits": 3, "on": ["rejected"]}
|
||||
```
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "acceptance_checks": "implementer.acceptance_checks", "changed_files": "implementer.changed_files", "implementation": "implementer.implementation_summary?"}
|
||||
```
|
||||
|
||||
|
||||
```output
|
||||
{ summary, changed_files, ui_applicable, screenshots, ui_review, platform_conventions, findings }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Decide whether this change touches a user-facing UI. If it does not (a backend,
|
||||
library, CLI, config or data-only change with no rendered surface), set
|
||||
ui_applicable=false, explain briefly in summary, and return skipped. Do not invent
|
||||
screenshots or visual work.
|
||||
|
||||
When the change does affect a user-facing UI, verify it visually:
|
||||
Read android/AGENTS.md first. You alone perform phone work while the runner holds
|
||||
this stage's declared resource lease. Use controlled fixtures for changed states,
|
||||
including human-decision sheets, on a scratch server/database and throwaway session.
|
||||
Never mutate an operator session or restart deployed services. Record original
|
||||
settings and connection/workspace; use the configured authoritative endpoint, not a
|
||||
hardcoded scratch default. Keep every shell/ADB operation foreground and awaited.
|
||||
Restore original settings and main's build before successful return, under the same
|
||||
lease. Report capture/restoration failures explicitly; do not approve incomplete
|
||||
verification or advertise readiness when cleanup/restoration is uncertain.
|
||||
Include resource_cleanup_confirmed in output: true only after confirmed restoration;
|
||||
false on any uncertainty (a release veto that quarantines ownership even if tools are idle).
|
||||
- Build and launch the application the way the repository actually does it, then
|
||||
capture screenshots of every relevant screen, state and viewport (default, empty,
|
||||
loading, error, long content, small and large sizes) with the target platform's
|
||||
screenshot tooling through shell. Prefer a headless or scripted capture when the
|
||||
project supports one. Save the images under the run directory and list the exact
|
||||
capture commands and image paths in screenshots.
|
||||
- Read the captured images and review them as a UI/UX designer: layout, spacing,
|
||||
alignment, typographic hierarchy, color and contrast, iconography, interaction
|
||||
and focus states, loading/empty/error states, accessibility, and consistency.
|
||||
- Judge the UI against established industry patterns and, above all, the native
|
||||
conventions of the platform the app targets (operating system, toolkit or web
|
||||
standards). Record the platform you reviewed against in platform_conventions,
|
||||
and any deviations and concrete fixes in findings. Infer the target platform from
|
||||
the task and the repository's actual UI toolkit when it is unstated.
|
||||
- Approve only when the screenshots show the task and the acceptance checks are met
|
||||
and the UI is correct, consistent and native to the target platform; otherwise
|
||||
return rejected with specific, actionable findings the implementer can fix.
|
||||
|
||||
Keep the capture pass proportionate: capture each changed state once (for example one
|
||||
collapsed and one expanded section, one run state, one stuck header) instead of every
|
||||
screen at multiple viewports or pan directions, and reuse what this run already captured
|
||||
rather than re-doing it.
|
||||
|
||||
If the change is UI-facing but screenshots genuinely cannot be captured
|
||||
(for example no runnable UI or no display), return blocked rather than approving.
|
||||
Do not edit files, and do not approve a UI you did not actually see.
|
||||
|
||||
Return rejected only for a must-fix visual defect with a concrete location, evidence
|
||||
and remedy; record non-blocking polish, duplication or convention notes in findings
|
||||
and return advisory, which continues the run without another implementer visit.
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-luna
|
||||
retries=1
|
||||
timeout=45m
|
||||
```
|
||||
|
||||
|
||||
```edges
|
||||
skipped -> committer
|
||||
approved -> committer
|
||||
advisory -> committer
|
||||
rejected -> implementer
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
|
||||
# stage committer
|
||||
|
||||
```memory
|
||||
{"task": "input.prompt", "changed_files": "implementer.changed_files", "review": "reviewer.summary?", "ui_review": "ui_reviewer.summary?"}
|
||||
```
|
||||
|
||||
```output
|
||||
{ commit, files, conflicts_resolved, summary }
|
||||
```
|
||||
|
||||
```prompt
|
||||
Commit the work this run produced, and resolve any conflict that is present. This stage
|
||||
runs after the reviews have approved and is the only stage allowed to commit.
|
||||
|
||||
Before committing, check that the change is reflected where it belongs: the packaged
|
||||
templates, the packaged
|
||||
authoring skill references. Update anything the change makes stale, and state in the
|
||||
commit body what you checked, saying explicitly when none of them needs updating.
|
||||
|
||||
Stage exactly the files this run changed - the paths in changed_files that still exist -
|
||||
using the repository's own version control commands, and commit them as one commit whose
|
||||
subject says what the change does and whose body summarises it. Do not stage unrelated
|
||||
files, generated or ignored files, or anything another change owns. Do not invent work
|
||||
that was not done; describe what is actually in the diff.
|
||||
|
||||
If the workspace is in a conflicted state - unmerged paths, or a merge, rebase or
|
||||
cherry-pick left unfinished - resolve each conflict: keep both intents where they are
|
||||
compatible, and choose the result that satisfies the task where they are not, then
|
||||
complete the operation so the tree is clean and the commit is finished. Never commit
|
||||
conflict markers, and never discard the other side's work silently; record what you
|
||||
resolved.
|
||||
|
||||
Do not push, tag, revert, or rewrite unrelated history. If there is nothing to commit,
|
||||
report that instead of creating an empty commit. Report the commit identifier, the files
|
||||
committed, whatever you resolved, and anything you deliberately left uncommitted.
|
||||
|
||||
|
||||
Finally, bring this branch up to date with main, in this same stage. After committing the work
|
||||
above, merge main into this branch here, resolve any conflict here - keeping both intents where they
|
||||
are compatible, and choosing the result that satisfies this task where they are not - run the
|
||||
repository's checks again after resolving, and commit the completed merge with a message naming what
|
||||
was combined. The point is that this branch can later be merged back without conflict: leave the
|
||||
tree clean and the merge finished. Never leave conflict markers, never discard the other side's work
|
||||
silently. If the merge genuinely cannot be completed, say so with the path and the reason instead of
|
||||
leaving the merge half-finished.
|
||||
|
||||
Then land it: this branch is the work and main is where it belongs, so merge this branch into main
|
||||
as the last act of this stage. Find the repository's main working tree with `git worktree list` (the
|
||||
entry checked out on main, not this one) and merge this branch into it there with `--no-ff`, with a
|
||||
message saying what the change does. Two lanes can land at the same moment, so take a lock first -
|
||||
`mkdir` a lock directory beside main's `git` directory in a bounded retry loop, and remove it when
|
||||
done - and retry briefly if another lane holds it. Then verify with
|
||||
`git merge-base --is-ancestor <this branch> main` and report the commit main now has. If main cannot
|
||||
be updated, say exactly why and leave this branch committed and clean rather than leaving main
|
||||
half-merged. There is no separate merge pipeline and nothing to hand this to: if it cannot be landed
|
||||
here, it is reported here.
|
||||
|
||||
|
||||
Do not edit README or other shared documentation files: an item records what it changed and which documentation is now stale in its own commit body, and the shared documents are folded by a single writer.
|
||||
|
||||
```
|
||||
|
||||
```tools
|
||||
read
|
||||
glob
|
||||
grep
|
||||
edit
|
||||
shell
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.6-terra
|
||||
retries=1
|
||||
timeout=30m
|
||||
```
|
||||
|
||||
```edges
|
||||
success -> #success
|
||||
blocked -> #failure
|
||||
error -> #failure
|
||||
```
|
||||
@@ -2,9 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -270,44 +268,6 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
response.Error = err.Error()
|
||||
}
|
||||
}
|
||||
} else if (request.Type == protocol.InspectResource || request.Type == protocol.ReconcileResource || request.Type == protocol.InspectResourcePool || request.Type == protocol.ConfigureResourcePool) && len(stores) > 0 {
|
||||
var err error
|
||||
response.Type = protocol.ResourceInspectionResult
|
||||
// Recovery has a separate transport path. A stage/model client may
|
||||
// also be local, so loopback alone is not an authority boundary.
|
||||
// The CLI uses /operator-connect; the ordinary /connect path is
|
||||
// deliberately refused even from loopback.
|
||||
if r.URL.Path != "/operator-connect" || !isLoopbackPeer(r.RemoteAddr) {
|
||||
response.Type = protocol.Error
|
||||
response.Error = "resource recovery requires the local operator transport"
|
||||
} else if request.Type == protocol.InspectResourcePool {
|
||||
response.Type = protocol.ResourcePoolInspection
|
||||
var pool protocol.ResourcePool
|
||||
pool, err = stores[0].InspectResourcePool(ctx, request.Resource)
|
||||
response.ResourcePool = &pool
|
||||
} else if request.Type == protocol.ConfigureResourcePool {
|
||||
response.Type = protocol.ResourcePoolConfigured
|
||||
if request.ResourcePool == nil {
|
||||
err = fmt.Errorf("resource pool declaration is required")
|
||||
} else {
|
||||
err = stores[0].ConfigureResourcePool(ctx, *request.ResourcePool)
|
||||
}
|
||||
} else if request.Type == protocol.InspectResource {
|
||||
var inspection protocol.ResourceInspection
|
||||
inspection, err = stores[0].InspectResource(ctx, request.Resource)
|
||||
response.ResourceInspection = &inspection
|
||||
} else {
|
||||
response.Type = protocol.ResourceReconciled
|
||||
if request.ResourceRecovery == nil {
|
||||
err = fmt.Errorf("resource recovery evidence is required")
|
||||
} else {
|
||||
err = stores[0].ReconcileResource(ctx, *request.ResourceRecovery)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
response.Type = protocol.Error
|
||||
response.Error = err.Error()
|
||||
}
|
||||
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence) {
|
||||
var err error
|
||||
switch request.Type {
|
||||
@@ -365,19 +325,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
}
|
||||
}
|
||||
mux.HandleFunc("GET /connect", connect)
|
||||
mux.HandleFunc("GET /operator-connect", connect)
|
||||
return &runHandler{Handler: mux, runner: runs}
|
||||
}
|
||||
|
||||
func isLoopbackPeer(remoteAddr string) bool {
|
||||
host, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
|
||||
type runHandler struct {
|
||||
http.Handler
|
||||
runner *runner
|
||||
|
||||
Reference in New Issue
Block a user