package server import ( "bytes" "context" "crypto/sha256" "encoding/json" "fmt" "net" "net/http/httptest" "os" "path/filepath" "reflect" "strings" "testing" "time" "git.a-wan.net/alan/pipelines/internal/cli" "git.a-wan.net/alan/pipelines/internal/client" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" ) type fixedRemoteListener struct{ net.Listener } func (l fixedRemoteListener) Accept() (net.Conn, error) { conn, err := l.Listener.Accept() if err != nil { return nil, err } return fixedRemoteConn{Conn: conn}, nil } type fixedRemoteConn struct{ net.Conn } func (fixedRemoteConn) RemoteAddr() net.Addr { return &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 443} } // This is a real CLI/WebSocket/scratch-SQLite exercise with an explicitly FAKE // executor. It never calls inference, a phone, or a deployed server. func TestResourceScratchCLIWaitingThenProgress(t *testing.T) { t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json")) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db")) if err != nil { t.Fatal(err) } defer store.Close() source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"fake-phone\"}\n```\n" if _, err := store.Submit(ctx, "scratch-resource", source); err != nil { t.Fatal(err) } handler := Handler(ctx, store).(*runHandler) started, release := make(chan struct{}), make(chan struct{}) handler.runner.execute = func(ctx context.Context, _ stage.Definition, input stage.Values) (stage.Result, error) { if input["prompt"] == "A" { close(started) select { case <-release: case <-ctx.Done(): return stage.Result{}, ctx.Err() } } return stage.Result{Outcome: "success", Output: stage.Values{"execution": "FAKE executor; no device or inference"}}, nil } handler.runner.events.interval = time.Millisecond events := make(chan protocol.RunEventFrame, 64) sub := handler.runner.events.subscribe("cli-evidence", "", nil, func(m protocol.Message) error { if m.RunEvent != nil { events <- *m.RunEvent } return nil }) sub.ready() defer sub.close() srv := httptest.NewServer(handler) defer srv.Close() defer func() { cancel(); WaitRuns(handler) }() endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" run := func(args ...string) string { var out bytes.Buffer app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil { t.Fatalf("CLI %v: %v\n%s", args, err, out.String()) } t.Logf("pipelines %s\n%s", strings.Join(args, " "), out.String()) return out.String() } parseRun := func(output string) int64 { var m protocol.Message if err := json.Unmarshal([]byte(output), &m); err != nil || m.RunDetail == nil { t.Fatal("CLI run response", output, err) } return m.RunDetail.ID } run("validate", "testdata/android-resource-draft.md", "--json") a := parseRun(run("run", "scratch-resource", "A", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "a")) <-started b := parseRun(run("run", "scratch-resource", "B", "--workspace", t.TempDir(), "--yes", "--json", "--request-key", "b")) awaitResourceEvent(t, events, b, func(e protocol.RunEventFrame) bool { return e.ResourceWait != nil }) var detail protocol.RunDetail if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.ResourceWait == nil || detail.Status != "queued" || len(detail.Steps) != 0 { t.Fatal("CLI did not observe queue", err) } if output := run("runs"); !strings.Contains(output, "Waiting for resource fake-phone") { t.Fatal("list omitted resource status", output) } var snapshot *protocol.RunEventFrame handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{b}}, handler.runner.events, handler.runner, "snapshot-evidence", func(m protocol.Message) error { if m.RunEvent != nil { snapshot = m.RunEvent } return nil }) handler.runner.events.unsubscribeClient("snapshot-evidence") if snapshot == nil || snapshot.ResourceWait == nil || !snapshot.ResourceWaitSet { t.Fatal("subscription snapshot omitted resource wait") } close(release) run("wait", fmt.Sprint(a), fmt.Sprint(b), "--json") detail = protocol.RunDetail{} if err := json.Unmarshal([]byte(run("inspect", fmt.Sprint(b), "--json")), &detail); err != nil || detail.Status != "succeeded" || detail.ResourceWait != nil || len(detail.Steps) != 1 { t.Fatal("automatic progress not observed", err) } t.Log("Observed queued resource wait followed by automatic progress; FAKE executor only, no phone/inference/deployment.") } func TestResourceOperatorTransportCLIRecoveryBoundary(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator.db")) if err != nil { t.Fatal(err) } defer store.Close() source := pipeline.Template([]string{"work"}, true) + "```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n" if _, err := store.Submit(ctx, "operator-resource", source); err != nil { t.Fatal(err) } items, err := store.List(ctx) if err != nil { t.Fatal(err) } detail, err := store.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } runID, _, err := store.CreateRun(ctx, detail, "operator-recovery", t.TempDir(), false, nil, nil) if err != nil { t.Fatal(err) } visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil { t.Fatalf("acquire resource: channel=%v err=%v", ch, err) } if err := store.RecoverRuns(ctx); err != nil { t.Fatal(err) } handler := Handler(ctx, store) srv := httptest.NewServer(handler) defer srv.Close() normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect") if err != nil { t.Fatal(err) } if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") { t.Fatalf("ordinary connect accepted recovery: %v", err) } normal.Close() endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" runCLI := func(args ...string) string { var out bytes.Buffer app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} if err := app.Run(ctx, append([]string{"--url", endpoint}, args...)); err != nil { t.Fatalf("CLI %v: %v\n%s", args, err, out.String()) } return out.String() } inspection := runCLI("resource-inspect", visit.Resource, "--json") if strings.Contains(inspection, "token") || strings.Contains(inspection, "credential") { t.Fatalf("inspection leaked capability: %s", inspection) } runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified no mutating action", "--yes", "--json") inspection = runCLI("resource-inspect", visit.Resource, "--json") var recovered protocol.ResourceInspection if err := json.Unmarshal([]byte(inspection), &recovered); err != nil { t.Fatal(err) } if recovered.Owner != nil || len(recovered.Audits) != 1 || recovered.Audits[0].RunID != runID || recovered.Audits[0].Result != "reconciled" { t.Fatalf("CLI recovery state was not persisted: %s", inspection) } } func TestResourceOperatorCLIRejectsStaleAndNonQuarantinedRecovery(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-rejections.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "operator-rejections", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil { t.Fatal(err) } items, err := store.List(ctx) if err != nil { t.Fatal(err) } detail, err := store.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } runID, _, err := store.CreateRun(ctx, detail, "operator-rejections", t.TempDir(), false, nil, nil) if err != nil { t.Fatal(err) } visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} lease, ch, _, err := store.RequestResource(ctx, visit) if err != nil || ch != nil { t.Fatalf("acquire resource: channel=%v err=%v", ch, err) } if err := store.RecoverRuns(ctx); err != nil { t.Fatal(err) } handler := Handler(ctx, store) srv := httptest.NewServer(handler) defer srv.Close() endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" runCLI := func(args ...string) error { var out bytes.Buffer app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} err := app.Run(ctx, append([]string{"--url", endpoint}, args...)) if strings.Contains(out.String(), "token") || strings.Contains(out.String(), "credential") { t.Fatalf("private capability appeared in CLI output: %s", out.String()) } return err } if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID+1), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { t.Fatal("accepted wrong owner") } if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), "stale-stage", fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { t.Fatal("accepted stale stage") } if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, "2", "--evidence", "verified", "--yes", "--json"); err == nil { t.Fatal("accepted stale sequence") } if err := store.SetResourceCleanup(ctx, lease, true); err != nil { t.Fatal(err) } if err := runCLI("resource-reconcile", visit.Resource, fmt.Sprint(runID), visit.Stage, fmt.Sprint(visit.Sequence), "--evidence", "verified", "--yes", "--json"); err == nil { t.Fatal("accepted non-quarantined owner") } inspection, err := store.InspectResource(ctx, visit.Resource) if err != nil { t.Fatal(err) } if inspection.Owner == nil || !inspection.Owner.CleanupConfirmed || len(inspection.Audits) != 0 { t.Fatalf("rejected recovery changed state: owner=%#v audits=%d", inspection.Owner, len(inspection.Audits)) } } func TestResourceOperatorTransportDoesNotTrustRemoteOrForwardedHeaders(t *testing.T) { for _, remote := range []string{"203.0.113.9:443", "198.51.100.7:8080", "not-an-address"} { if isLoopbackPeer(remote) { t.Fatalf("accepted non-loopback peer %q", remote) } } for _, remote := range []string{"127.0.0.1:443", "[::1]:443"} { if !isLoopbackPeer(remote) { t.Fatalf("rejected loopback peer %q", remote) } } // X-Forwarded-For is deliberately not consulted: a proxy-supplied header // cannot turn a remote WebSocket peer into the local operator transport. } func TestResourceOperatorTransportRejectsForgedForwardedPeer(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "operator-transport.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "operator-transport", pipeline.Template([]string{"work"}, true)+"```resources\n{\"exclusive\":\"scratch-resource\"}\n```\n"); err != nil { t.Fatal(err) } items, err := store.List(ctx) if err != nil { t.Fatal(err) } detail, err := store.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } runID, _, err := store.CreateRun(ctx, detail, "operator-transport", t.TempDir(), false, nil, nil) if err != nil { t.Fatal(err) } visit := pipeline.ResourceVisit{Resource: "scratch-resource", RunID: runID, Stage: "work", Sequence: 1} if _, ch, _, err := store.RequestResource(ctx, visit); err != nil || ch != nil { t.Fatalf("acquire resource: channel=%v err=%v", ch, err) } if err := store.RecoverRuns(ctx); err != nil { t.Fatal(err) } before, err := store.InspectResource(ctx, visit.Resource) if err != nil { t.Fatal(err) } server := httptest.NewUnstartedServer(Handler(ctx, store)) server.Listener = fixedRemoteListener{Listener: server.Listener} server.Start() defer server.Close() conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/operator-connect", &websocket.DialOptions{HTTPHeader: map[string][]string{"X-Forwarded-For": {"127.0.0.1"}}}) if err != nil { t.Fatal(err) } defer conn.CloseNow() var welcome protocol.Message if err := wsjson.Read(ctx, conn, &welcome); err != nil { t.Fatal(err) } if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource, ID: "forged"}); err != nil { t.Fatal(err) } var refused protocol.Message if err := wsjson.Read(ctx, conn, &refused); err != nil { t.Fatal(err) } if refused.Type != protocol.Error || !strings.Contains(refused.Error, "operator transport") { t.Fatalf("forged peer was accepted: %#v", refused) } after, err := store.InspectResource(ctx, visit.Resource) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(before, after) { t.Fatalf("forged request mutated recovery state: before=%#v after=%#v", before, after) } trusted := httptest.NewServer(Handler(ctx, store)) defer trusted.Close() normal, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/connect") if err != nil { t.Fatal(err) } if _, err := normal.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err == nil || !strings.Contains(err.Error(), "operator transport") { t.Fatalf("ordinary connect accepted recovery: %v", err) } normal.Close() operator, err := client.Open(ctx, "ws"+strings.TrimPrefix(trusted.URL, "http")+"/operator-connect") if err != nil { t.Fatal(err) } defer operator.Close() if _, err := operator.Call(ctx, protocol.Message{Type: protocol.InspectResource, Resource: visit.Resource}); err != nil { t.Fatalf("trusted operator transport refused inspection: %v", err) } } func TestResourcePoolOperatorCLIConfigureReadbackAndPreserveLease(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() dbPath := filepath.Join(t.TempDir(), "pool-operator.db") store, err := pipeline.Open(dbPath) if err != nil { t.Fatal(err) } defer store.Close() if err := store.ConfigureResourcePool(ctx, protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true, Metadata: map[string]string{"serial": "old"}}}}); err != nil { t.Fatal(err) } if _, err := store.Submit(ctx, "pool-owner", pipeline.Template([]string{"work"}, true)+"```resources\n{\"pool\":\"targets\"}\n```\n"); err != nil { t.Fatal(err) } items, err := store.List(ctx) if err != nil || len(items) != 1 { t.Fatal("pool pipeline missing", err) } detail, err := store.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } runID, _, err := store.CreateRun(ctx, detail, "pool-owner", t.TempDir(), false, nil, nil) if err != nil { t.Fatal(err) } lease, ready, _, err := store.RequestResourcePool(ctx, "targets", pipeline.ResourceVisit{Resource: "targets", RunID: runID, Stage: "work", Sequence: 1}) if err != nil || ready != nil { t.Fatalf("pool lease: ready=%v err=%v", ready, err) } handler := Handler(ctx, store) srv := httptest.NewServer(handler) defer srv.Close() endpoint := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" normal, err := client.Open(ctx, endpoint) if err != nil { t.Fatal(err) } if _, err := normal.Call(ctx, protocol.Message{Type: protocol.ConfigureResourcePool, ResourcePool: &protocol.ResourcePool{Name: "targets", Members: []protocol.ResourcePoolMember{{Member: "phone", Enabled: true}}}}); err == nil || !strings.Contains(err.Error(), "operator transport") { t.Fatalf("remote configuration was accepted: %v", err) } normal.Close() declaration := filepath.Join(t.TempDir(), "pool.json") if err := os.WriteFile(declaration, []byte(`{"name":"targets","members":[{"member":"phone","enabled":true,"metadata":{"serial":"new"}},{"member":"emulator","enabled":false}]}`), 0600); err != nil { t.Fatal(err) } var out bytes.Buffer for _, fromStdin := range []bool{false, true} { oversized := strings.Repeat(" ", 256*1024+1) path := filepath.Join(t.TempDir(), "oversized.json") if err := os.WriteFile(path, []byte(oversized), 0600); err != nil { t.Fatal(err) } if fromStdin { path = "-" } bounded := cli.App{Out: &out, Err: &out, In: strings.NewReader(oversized)} err := bounded.Run(ctx, []string{"--url", endpoint, "resource-pool-configure", path, "--yes", "--json"}) if err == nil || !strings.Contains(err.Error(), "exceeds 262144 bytes") { t.Fatalf("oversized pool input (stdin=%v): %v", fromStdin, err) } out.Reset() } app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} if err := app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http"), "resource-pool-configure", declaration, "--yes", "--json"}); err != nil { t.Fatalf("configure CLI: %v\n%s", err, out.String()) } out.Reset() app = cli.App{Out: &out, Err: &out, In: strings.NewReader("")} if err := app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http"), "resource-pool-inspect", "targets", "--json"}); err != nil { t.Fatalf("inspect CLI: %v\n%s", err, out.String()) } var pool protocol.ResourcePool if err := json.Unmarshal(out.Bytes(), &pool); err != nil || len(pool.Members) != 2 || pool.Members[0].Metadata["serial"] != "new" { t.Fatalf("bad pool readback: %s (%v)", out.String(), err) } inspection, err := store.InspectResource(ctx, "phone") if err != nil || inspection.Owner == nil || inspection.Owner.RunID != runID { t.Fatalf("configuration released active lease: %#v (%v)", inspection, err) } if err := store.SetResourceCleanup(ctx, lease, true); err != nil { t.Fatal(err) } if err := store.ReleaseResource(ctx, lease); err != nil { t.Fatal(err) } srv.Close() store.Close() reopened, err := pipeline.Open(dbPath) if err != nil { t.Fatal(err) } defer reopened.Close() persisted, err := reopened.InspectResourcePool(ctx, "targets") if err != nil || persisted.Members[0].Metadata["serial"] != "new" { t.Fatalf("pool did not survive reopen: %#v (%v)", persisted, err) } } func TestResourceAndroidDraftPublishedV3Provenance(t *testing.T) { baseline, err := os.ReadFile("testdata/android-v3-published.md") if err != nil { t.Fatal(err) } if hash := fmt.Sprintf("%x", sha256.Sum256(baseline)); hash != "4a3a4b8b947eb20c115c5470e6e378f117d5fa291a7b6c8ae966a3745bc826de" { t.Fatal("baseline is not retrieved Android v3", hash) } draft, err := os.ReadFile("testdata/android-resource-draft.md") if err != nil { t.Fatal(err) } old, err := pipeline.Parse(string(baseline)) if err != nil { t.Fatal(err) } updated, err := pipeline.Parse(string(draft)) if err != nil { t.Fatal(err) } if len(old.Stages) != len(updated.Stages) { t.Fatal("draft changed graph shape") } for i := range old.Stages { a, b := old.Stages[i], updated.Stages[i] if b.Configs["provider"] != "openai" || b.Configs["model"] != a.Configs["model"] { t.Fatal("draft changed approved model") } if b.Key == "ui_reviewer" { if b.Resources == nil || b.Resources.Exclusive != "phone-47241FDAS00DLK" { t.Fatal("reviewer resource missing") } } else if b.Resources != nil { t.Fatal("non-UI stage acquired phone") } a.Prompt, b.Prompt = "", "" a.Resources, b.Resources = nil, nil if !reflect.DeepEqual(a, b) { t.Fatal("draft changed fields outside approved role split/resource", a.Key) } } }