Files
pipelines/internal/server/resource_cli_test.go
T

512 lines
19 KiB
Go

package server
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/client"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
type fixedRemoteListener struct{ net.Listener }
func (l fixedRemoteListener) Accept() (net.Conn, error) {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return fixedRemoteConn{Conn: conn}, nil
}
type fixedRemoteConn struct{ net.Conn }
func (fixedRemoteConn) RemoteAddr() net.Addr {
return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443}
}
// This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE
// executor. It never calls inference, a phone, or a deployed server.
func TestResourceScratchCLIWaitingThenProgress(t *testing.T) {
t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json"))
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n"
if _, err := store.Submit(ctx, "scratch-resource", source); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store).(*runHandler)
started, release := make(chan struct{}), make(chan struct{})
handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) {
if input["prompt"] == "A" {
close(started)
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
}
return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil
}
handler.runner.events.interval = time.Millisecond
events := make(chan protocol.RunEventFrame, 64)
sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error {
if m.RunEvent != nil {
events <- *m.RunEvent
}
return nil
})
sub.ready()
defer sub.close()
srv := httptest.NewServer(handler)
defer srv.Close()
defer func() { cancel(); WaitRuns(handler) }()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
run := func(args ...string) string {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
}
t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String())
return out.String()
}
parseRun := func(output string) int64 {
var m protocol.Message
if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil {
t.Fatal("CLI run response", output, err)
}
return m.RunDetail.ID
}
run("validate", "testdata/android-resource-draft.md", "--json")
a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a"))
<-started
b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b"))
awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil })
var detail protocol.RunDetail
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 {
t.Fatal("CLI did not observe queue", err)
}
if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") {
t.Fatal("list omitted resource status", output)
}
var snapshot *protocol.RunEventFrame
handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error {
if m.RunEvent != nil {
snapshot = m.RunEvent
}
return nil
})
handler.runner.events.unsubscribeClient("snapshot-evidence")
if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet {
t.Fatal("subscription snapshot omitted resource wait")
}
close(release)
run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json")
detail = protocol.RunDetail{}
if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 {
t.Fatal("automatic progress not observed", err)
}
t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.")
}
func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"
if _, err := store.Submit(ctx, "operator-resource", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) string {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil {
t.Fatalf("CLI %v: %v\n%s", args, err, out.String())
}
return out.String()
}
inspection := runCLI("resource-inspect", visit.Resource, "--json")
if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") {
t.Fatalf("inspection leaked capability: %s", inspection)
}
runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json")
inspection = runCLI("resource-inspect", visit.Resource, "--json")
var recovered protocol.ResourceInspection
if err := json.Unmarshal([]byte(inspection), &recovered); err != nil {
t.Fatal(err)
}
if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" {
t.Fatalf("CLI recovery state was not persisted: %s", inspection)
}
}
func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
lease, ch, _, err := store.RequestResource(ctx, visit)
if err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
runCLI := func(args ...string) error {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
err := app.Run(ctx, append([]string{"--url", endpoint}, args...))
if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") {
t.Fatalf("private capability appeared in CLI output: %s", out.String())
}
return err
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted wrong owner")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale stage")
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted stale sequence")
}
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
t.Fatal(err)
}
if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil {
t.Fatal("accepted non-quarantined owner")
}
inspection, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 {
t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits))
}
}
func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) {
for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} {
if isLoopbackPeer(remote) {
t.Fatalf("accepted non-loopback peer %q", remote)
}
}
for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} {
if !isLoopbackPeer(remote) {
t.Fatalf("rejected loopback peer %q", remote)
}
}
// X-Forwarded-For is deliberately not consulted: a proxy-supplied header
// cannot turn a remote WebSocket peer into the local operator transport.
}
func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1}
if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil {
t.Fatalf("acquire resource: channel=%v err=%v", ch, err)
}
if err := store.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
before, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
server := httptest.NewUnstartedServer(Handler(ctx, store))
server.Listener = fixedRemoteListener{Listener: server.Listener}
server.Start()
defer server.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}})
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
var welcome protocol.Message
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
t.Fatal(err)
}
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil {
t.Fatal(err)
}
var refused protocol.Message
if err := wsjson.Read(ctx, conn, &refused); err != nil {
t.Fatal(err)
}
if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") {
t.Fatalf("forged peer was accepted: %#v", refused)
}
after, err := store.InspectResource(ctx, visit.Resource)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(before, after) {
t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after)
}
trusted := httptest.NewServer(Handler(ctx, store))
defer trusted.Close()
normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect")
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("ordinary connect accepted recovery: %v", err)
}
normal.Close()
operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect")
if err != nil {
t.Fatal(err)
}
defer operator.Close()
if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil {
t.Fatalf("trusted operator transport refused inspection: %v", err)
}
}
func TestResourcePoolOperatorCLIConfigureReadbackAndPreserveLease(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
dbPath := filepath.Join(t.TempDir(), "pool-operator.db")
store, err := pipeline.Open(dbPath)
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "old"}}}}); err != nil {
t.Fatal(err)
}
if _, err := store.Submit(ctx, "pool-owner", pipeline.Template([]string{"work"}, true)+"```resources\n{\"pool\":\"targets\"}\n```\n"); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil || len(items) != 1 {
t.Fatal("pool pipeline missing", err)
}
detail, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
runID, _, err := store.CreateRun(ctx, detail, "pool-owner", t.TempDir(), false, nil, nil)
if err != nil {
t.Fatal(err)
}
lease, ready, _, err := store.RequestResourcePool(ctx, "targets", pipeline.ResourceVisit{Resource: "targets", RunID: runID, Stage: "work", Sequence: 1})
if err != nil || ready != nil {
t.Fatalf("pool lease: ready=%v err=%v", ready, err)
}
handler := Handler(ctx, store)
srv := httptest.NewServer(handler)
defer srv.Close()
endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
normal, err := client.Open(ctx, endpoint)
if err != nil {
t.Fatal(err)
}
if _, err := normal.Call(ctx, protocol.Message{Type: protocol.ConfigureResourcePool, ResourcePool: &protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true}}}}); err == nil || !strings.Contains(err.Error(), "operator transport") {
t.Fatalf("remote configuration was accepted: %v", err)
}
normal.Close()
declaration := filepath.Join(t.TempDir(), "pool.json")
if err := os.WriteFile(declaration, []byte(`{"name":"targets","members":[{"member":"phone","enabled":true,"metadata":{"serial":"new"}},{"member":"emulator","enabled":false}]}`), 0600); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
for _, fromStdin := range []bool{false, true} {
oversized := strings.Repeat(" ", 256*1024+1)
path := filepath.Join(t.TempDir(), "oversized.json")
if err := os.WriteFile(path, []byte(oversized), 0600); err != nil {
t.Fatal(err)
}
if fromStdin {
path = "-"
}
bounded := cli.App{Out: &out, Err: &out, In: strings.NewReader(oversized)}
err := bounded.Run(ctx, []string{"--url", endpoint, "resource-pool-configure", path, "--yes", "--json"})
if err == nil || !strings.Contains(err.Error(), "exceeds 262144 bytes") {
t.Fatalf("oversized pool input (stdin=%v): %v", fromStdin, err)
}
out.Reset()
}
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http"), "resource-pool-configure", declaration, "--yes", "--json"}); err != nil {
t.Fatalf("configure CLI: %v\n%s", err, out.String())
}
out.Reset()
app = cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
if err := app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http"), "resource-pool-inspect", "targets", "--json"}); err != nil {
t.Fatalf("inspect CLI: %v\n%s", err, out.String())
}
var pool protocol.ResourcePool
if err := json.Unmarshal(out.Bytes(), &pool); err != nil || len(pool.Members) != 2 || pool.Members[0].Metadata["serial"] != "new" {
t.Fatalf("bad pool readback: %s (%v)", out.String(), err)
}
inspection, err := store.InspectResource(ctx, "phone")
if err != nil || inspection.Owner == nil || inspection.Owner.RunID != runID {
t.Fatalf("configuration released active lease: %#v (%v)", inspection, err)
}
if err := store.SetResourceCleanup(ctx, lease, true); err != nil {
t.Fatal(err)
}
if err := store.ReleaseResource(ctx, lease); err != nil {
t.Fatal(err)
}
srv.Close()
store.Close()
reopened, err := pipeline.Open(dbPath)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
persisted, err := reopened.InspectResourcePool(ctx, "targets")
if err != nil || persisted.Members[0].Metadata["serial"] != "new" {
t.Fatalf("pool did not survive reopen: %#v (%v)", persisted, err)
}
}
func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) {
baseline, err := os.ReadFile("testdata/android-v3-published.md")
if err != nil {
t.Fatal(err)
}
if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" {
t.Fatal("baseline is not retrieved Android v3", hash)
}
draft, err := os.ReadFile("testdata/android-resource-draft.md")
if err != nil {
t.Fatal(err)
}
old, err := pipeline.Parse(string(baseline))
if err != nil {
t.Fatal(err)
}
updated, err := pipeline.Parse(string(draft))
if err != nil {
t.Fatal(err)
}
if len(old.Stages) != len(updated.Stages) {
t.Fatal("draft changed graph shape")
}
for i := range old.Stages {
a, b := old.Stages[i], updated.Stages[i]
if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] {
t.Fatal("draft changed approved model")
}
if b.Key == "ui_reviewer" {
if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" {
t.Fatal("reviewer resource missing")
}
} else if b.Resources != nil {
t.Fatal("non-UI stage acquired phone")
}
a.Prompt, b.Prompt = "", ""
a.Resources, b.Resources = nil, nil
if !reflect.DeepEqual(a, b) {
t.Fatal("draft changed fields outside approved role split/resource", a.Key)
}
}
}