Files
pipelines/internal/server/chat_multiclient_test.go
T
alan e9d902213e Preserve background shell visibility across readback races
Reconcile authoritative shell snapshots with bounded lifecycle journals so concurrent exits and creates cannot be overwritten, coalesce observation setup, and retain verified ownership across reconnects. Add deterministic reconciliation and cancellation regressions plus stable limited-visibility handling.\n\nChecked packaged templates and authoring skill references: none need updating. The task-owned docs note records the stale shared chat documentation for docs-fold; shared documentation was not edited.
2026-10-18 12:54:36 +01:00

475 lines
20 KiB
Go

package server
import (
"context"
"errors"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// channelChat is a broker whose stream is driven frame-by-frame from a channel,
// so a test can hold a turn in flight and prove prompt serialisation, fan-out and
// exactly-once approval behaviour without a real OpenCode service.
type channelChat struct {
mu sync.Mutex
prompts []string
perm []protocol.ChatPermissionFrame
replies []protocol.ChatReplyRequest
frames chan protocol.ChatEventFrame
ready chan struct{}
once sync.Once
// promptErrAt, when nonzero, is the 1-based index of the Prompt call that
// fails, so a test can prove a broker failure is reported to the client that
// sent the prompt.
promptErrAt int
}
func newChannelChat() *channelChat {
return &channelChat{frames: make(chan protocol.ChatEventFrame, 32), ready: make(chan struct{})}
}
func (c *channelChat) Prompt(_ context.Context, _, text string) error {
c.mu.Lock()
c.prompts = append(c.prompts, text)
fail := c.promptErrAt != 0 && len(c.prompts) == c.promptErrAt
c.mu.Unlock()
if fail {
return errors.New("broker rejected the prompt")
}
return nil
}
func (c *channelChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
c.once.Do(func() {
close(c.ready)
if ready != nil {
ready()
}
})
for {
select {
case <-ctx.Done():
return ctx.Err()
case frame := <-c.frames:
if err := emit(frame); err != nil {
return err
}
}
}
}
func (c *channelChat) Interrupt(context.Context, string) error { return nil }
func (c *channelChat) Messages(context.Context, string, int) ([]protocol.ChatMessage, error) {
return nil, nil
}
func (c *channelChat) Sessions(context.Context) ([]protocol.ChatSession, error) { return nil, nil }
func (c *channelChat) Create(_ context.Context, title, directory string) (protocol.ChatSession, error) {
return protocol.ChatSession{ID: "ses_created", Title: title, Directory: directory}, nil
}
func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bool, error) {
return protocol.ChatSession{}, false, nil
}
func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil }
func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) {
c.mu.Lock()
defer c.mu.Unlock()
return append([]protocol.ChatPermissionFrame(nil), c.perm...), nil
}
func (c *channelChat) Reply(_ context.Context, request protocol.ChatReplyRequest) error {
c.mu.Lock()
c.replies = append(c.replies, request)
c.mu.Unlock()
return nil
}
func (c *channelChat) push(frame protocol.ChatEventFrame) {
c.frames <- frame
}
func (c *channelChat) promptCount() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.prompts)
}
func (c *channelChat) promptTexts() []string {
c.mu.Lock()
defer c.mu.Unlock()
return append([]string(nil), c.prompts...)
}
func (c *channelChat) replyCount() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.replies)
}
// dialChatServer starts one server and dials n connections, returning the server
// so a test can reconnect a dropped client to the same hub.
func dialChatServer(t *testing.T, options Options, n int) (context.Context, context.CancelFunc, *httptest.Server, []*websocket.Conn) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
srv := httptest.NewServer(HandlerWithOptions(ctx, options))
t.Cleanup(func() { srv.Close(); cancel() })
conns := make([]*websocket.Conn, 0, n)
for i := 0; i < n; i++ {
conns = append(conns, dialChatConn(t, ctx, srv))
}
return ctx, cancel, srv, conns
}
func dialChatConn(t *testing.T, ctx context.Context, srv *httptest.Server) *websocket.Conn {
t.Helper()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { conn.CloseNow() })
return conn
}
// readUntil reads messages until one of type want arrives, failing on a server
// error frame; interleaved push frames are skipped.
func readUntil(t *testing.T, ctx context.Context, conn *websocket.Conn, want string) protocol.Message {
t.Helper()
for {
var message protocol.Message
if err := wsjson.Read(ctx, conn, &message); err != nil {
t.Fatalf("read (waiting for %s): %v", want, err)
}
if message.Type == protocol.Error {
t.Fatalf("server error frame while waiting for %s: %s", want, message.Error)
}
if message.Type == want {
return message
}
}
}
// nextEvent reads the next chat_event frame strictly, so a test can assert the
// order of the turn's frames.
func nextEvent(t *testing.T, ctx context.Context, conn *websocket.Conn) protocol.ChatEventFrame {
t.Helper()
for {
var message protocol.Message
if err := wsjson.Read(ctx, conn, &message); err != nil {
t.Fatalf("read event: %v", err)
}
if message.Type == protocol.Error {
t.Fatalf("server error frame: %s", message.Error)
}
if message.Type == protocol.ChatEvent && message.ChatEvent != nil {
return *message.ChatEvent
}
}
}
// nextPermission reads the next chat_permission frame for requestID.
func nextPermission(t *testing.T, ctx context.Context, conn *websocket.Conn, requestID string) protocol.ChatPermissionFrame {
t.Helper()
for {
var message protocol.Message
if err := wsjson.Read(ctx, conn, &message); err != nil {
t.Fatalf("read permission: %v", err)
}
if message.Type == protocol.Error {
t.Fatalf("server error frame: %s", message.Error)
}
if message.Type == protocol.ChatPermission && message.ChatPermission != nil && message.ChatPermission.RequestID == requestID {
return *message.ChatPermission
}
}
}
func writeChat(t *testing.T, ctx context.Context, conn *websocket.Conn, message protocol.Message) {
t.Helper()
if err := wsjson.Write(ctx, conn, message); err != nil {
t.Fatal(err)
}
}
// TestChatFanoutToSeveralClients is the multi-client acceptance test: a prompt
// from one client is accepted, its user turn is pushed live to the other client
// attributed to the sending device, and the turn's streaming frames reach every
// watching client, not only the one that prompted.
func TestChatFanoutToSeveralClients(t *testing.T) {
broker := newChannelChat()
ctx, _, _, conns := dialChatServer(t, Options{Chat: broker}, 2)
phone, tablet := conns[0], conns[1]
// The tablet watches the session before anything is sent.
writeChat(t, ctx, tablet, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, ctx, tablet, protocol.ChatSubscribe)
// The phone prompts. Its ack is not a queue acknowledgement.
writeChat(t, ctx, phone, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "from the phone", Client: "Pixel 9"}})
ack := readUntil(t, ctx, phone, protocol.ChatPrompt)
if ack.ID != "p1" || ack.ChatQueued {
t.Fatalf("phone prompt ack = %+v, want accepted (not queued)", ack)
}
// The tablet sees the phone's message, attributed to the phone.
userFrame := nextEvent(t, ctx, tablet)
if userFrame.Kind != protocol.ChatEventUser || userFrame.Client != "Pixel 9" || userFrame.Part == nil || userFrame.Part.Text != "from the phone" {
t.Fatalf("tablet user frame = %+v, want the phone's message attributed to Pixel 9", userFrame)
}
// The turn's streaming output reaches both clients in order.
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", MessageID: "msg_1", Kind: protocol.ChatEventText, Delta: "hel", Part: &protocol.ChatPart{Type: protocol.ChatPartText, ID: "msg_1#0"}})
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", MessageID: "msg_1", Kind: protocol.ChatEventText, Delta: "lo", Part: &protocol.ChatPart{Type: protocol.ChatPartText, ID: "msg_1#0"}})
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", MessageID: "msg_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, Index: 1}})
for _, client := range []struct {
name string
conn *websocket.Conn
}{{"phone", phone}, {"tablet", tablet}} {
var text strings.Builder
for i := 0; i < 2; i++ {
frame := nextEvent(t, ctx, client.conn)
if frame.Kind != protocol.ChatEventText {
t.Fatalf("%s frame %d = %+v, want text", client.name, i, frame)
}
text.WriteString(frame.Delta)
}
if text.String() != "hello" {
t.Fatalf("%s saw %q, want the streamed turn output", client.name, text.String())
}
if end := nextEvent(t, ctx, client.conn); end.Kind != protocol.ChatEventStep || end.Step == nil || end.Step.Status != protocol.ChatStepEnded {
t.Fatalf("%s step frame = %+v, want ended", client.name, end)
}
}
}
// TestChatPromptSerialisedPerSession pins that two prompts for one session never
// interleave: the second is accepted as queued and only posted once the first
// turn ends.
func TestChatPromptSerialisedPerSession(t *testing.T) {
broker := newChannelChat()
ctx, _, _, conns := dialChatServer(t, Options{Chat: broker}, 2)
phone, tablet := conns[0], conns[1]
writeChat(t, ctx, phone, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "one"}})
if ack := readUntil(t, ctx, phone, protocol.ChatPrompt); ack.ChatQueued {
t.Fatalf("first prompt should start immediately: %+v", ack)
}
// The tablet's prompt while the turn is in flight is queued, and it is told so.
writeChat(t, ctx, tablet, protocol.Message{Type: protocol.ChatPrompt, ID: "p2", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "two"}})
if ack := readUntil(t, ctx, tablet, protocol.ChatPrompt); !ack.ChatQueued {
t.Fatalf("second prompt should be acknowledged as queued: %+v", ack)
}
if broker.promptCount() != 1 {
t.Fatalf("prompts forwarded = %d, want 1 while the first turn is in flight", broker.promptCount())
}
// End the first turn; the queued prompt is then posted.
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", MessageID: "msg_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, Index: 1}})
deadline := time.Now().Add(3 * time.Second)
for broker.promptCount() < 2 {
if time.Now().After(deadline) {
t.Fatalf("queued prompt was never posted: %v", broker.promptTexts())
}
time.Sleep(5 * time.Millisecond)
}
if got := broker.promptTexts(); len(got) != 2 || got[0] != "one" || got[1] != "two" {
t.Fatalf("prompts = %v, want [one two] in order", got)
}
}
// TestChatApprovalFirstAnswerWins is the exactly-once acceptance test: a pending
// approval is shared, the first answer is recorded with the client that gave it
// and broadcast to every client, and a later answer is refused.
func TestChatApprovalFirstAnswerWins(t *testing.T) {
broker := newChannelChat()
broker.perm = []protocol.ChatPermissionFrame{{SessionID: "ses_1", RequestID: "req_1", Action: "bash", Resources: []string{"rm -rf /tmp"}}}
ctx, _, _, conns := dialChatServer(t, Options{Chat: broker}, 2)
phone, tablet := conns[0], conns[1]
// A turn starts so the poller runs and pushes the pending request.
writeChat(t, ctx, phone, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "run it"}})
readUntil(t, ctx, phone, protocol.ChatPrompt)
pending := nextPermission(t, ctx, phone, "req_1")
if pending.Resolved {
t.Fatalf("request already marked resolved: %+v", pending)
}
// A second client attaches and is replayed the still-pending request, so its
// card is answerable too.
writeChat(t, ctx, tablet, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, ctx, tablet, protocol.ChatSubscribe)
replay := nextPermission(t, ctx, tablet, "req_1")
if replay.Resolved || replay.Action != "bash" {
t.Fatalf("replayed permission = %+v, want the pending request", replay)
}
// The phone answers first.
writeChat(t, ctx, phone, protocol.Message{Type: protocol.ChatReply, ID: "a1", ChatReply: &protocol.ChatReplyRequest{SessionID: "ses_1", RequestID: "req_1", Decision: protocol.ChatDecisionOnce, Client: "Pixel 9"}})
readUntil(t, ctx, phone, protocol.ChatReply)
// Both clients see the resolution, with the answering device recorded.
for _, client := range []struct {
name string
conn *websocket.Conn
}{{"phone", phone}, {"tablet", tablet}} {
resolution := nextPermission(t, ctx, client.conn, "req_1")
if !resolution.Resolved || resolution.Decision != protocol.ChatDecisionOnce || resolution.ResolvedBy != "Pixel 9" {
t.Fatalf("%s resolution = %+v, want resolved once by Pixel 9", client.name, resolution)
}
}
// The tablet's later answer is refused, and it never reaches the agent.
writeChat(t, ctx, tablet, protocol.Message{Type: protocol.ChatReply, ID: "a2", ChatReply: &protocol.ChatReplyRequest{SessionID: "ses_1", RequestID: "req_1", Decision: protocol.ChatDecisionAlways, Client: "Pixel 10 Pro Fold"}})
var refused protocol.Message
if err := wsjson.Read(ctx, tablet, &refused); err != nil {
t.Fatal(err)
}
if refused.Type != protocol.Error || !strings.Contains(refused.Error, "already answered by Pixel 9") {
t.Fatalf("second answer was not refused as already answered: %+v", refused)
}
if broker.replyCount() != 1 {
t.Fatalf("broker replies = %d, want the agent to receive exactly one answer", broker.replyCount())
}
}
// TestChatSubscribeWireShape pins the exact wire body the Android app sends for
// chat_subscribe (ChatProtocol.subscribeRequest): the session id travels in the
// chat_subscribe field, not chat_query. The handler once read chat_query, so the
// unit tests passed while the shipped app's subscribe was silently rejected; this
// test writes the raw JSON, attaches the connection and proves a prompt from
// another client reaches it.
func TestChatSubscribeWireShape(t *testing.T) {
broker := newChannelChat()
ctx, _, _, conns := dialChatServer(t, Options{Chat: broker}, 2)
phone, watch := conns[0], conns[1]
if err := watch.Write(ctx, websocket.MessageText, []byte(`{"type":"chat_subscribe","id":"s1","chat_subscribe":{"session_id":"ses_1"}}`)); err != nil {
t.Fatal(err)
}
if ack := readUntil(t, ctx, watch, protocol.ChatSubscribe); ack.ID != "s1" {
t.Fatalf("subscribe ack = %+v, want id s1", ack)
}
// The watcher is attached: another client's prompt is fanned out to it.
writeChat(t, ctx, phone, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "ping", Client: "Pixel 9"}})
readUntil(t, ctx, phone, protocol.ChatPrompt)
frame := nextEvent(t, ctx, watch)
if frame.Kind != protocol.ChatEventUser || frame.Client != "Pixel 9" || frame.Part == nil || frame.Part.Text != "ping" {
t.Fatalf("watcher frame = %+v, want the phone's message (subscribe did not attach)", frame)
}
}
// TestChatQueuedPromptFailureSurfaced pins that a prompt accepted as queued whose
// delivery to the agent later fails is reported to the client that sent it,
// instead of leaving its composer showing "Queued" with no error.
func TestChatQueuedPromptFailureSurfaced(t *testing.T) {
broker := newChannelChat()
broker.promptErrAt = 2 // the queued (second) prompt fails to reach the agent
ctx, _, _, conns := dialChatServer(t, Options{Chat: broker}, 2)
phone, tablet := conns[0], conns[1]
// The phone's prompt starts a turn held in flight.
writeChat(t, ctx, phone, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "one"}})
if ack := readUntil(t, ctx, phone, protocol.ChatPrompt); ack.ChatQueued {
t.Fatalf("first prompt should start immediately: %+v", ack)
}
// The tablet's prompt is accepted as queued and acknowledged as such.
writeChat(t, ctx, tablet, protocol.Message{Type: protocol.ChatPrompt, ID: "p2", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "two"}})
if ack := readUntil(t, ctx, tablet, protocol.ChatPrompt); !ack.ChatQueued {
t.Fatalf("second prompt should be acknowledged as queued: %+v", ack)
}
// End the first turn; the queued prompt is posted and the broker rejects it.
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", MessageID: "msg_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, Index: 1}})
for {
var message protocol.Message
if err := wsjson.Read(ctx, tablet, &message); err != nil {
t.Fatalf("read (waiting for the queued prompt failure): %v", err)
}
if message.Type != protocol.Error {
continue
}
if message.ID != "p2" || !strings.Contains(message.Error, "broker rejected the prompt") {
t.Fatalf("queued prompt failure = %+v, want an error for p2 naming the broker failure", message)
}
break
}
}
// TestChatPendingApprovalsAllReplayed pins that every unresolved approval request
// a room holds is replayed to a client that attaches afterwards, not only the
// most recent one, so both cards are answerable on the late client.
func TestChatPendingApprovalsAllReplayed(t *testing.T) {
broker := newChannelChat()
broker.perm = []protocol.ChatPermissionFrame{
{SessionID: "ses_1", RequestID: "req_1", Action: "bash", Resources: []string{"rm -rf /tmp"}},
{SessionID: "ses_1", RequestID: "req_2", Action: "write", Resources: []string{"/etc/hosts"}},
}
ctx, _, _, conns := dialChatServer(t, Options{Chat: broker}, 2)
phone, tablet := conns[0], conns[1]
// A turn starts so the poller pushes both pending requests to the phone.
writeChat(t, ctx, phone, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "run them"}})
readUntil(t, ctx, phone, protocol.ChatPrompt)
nextPermission(t, ctx, phone, "req_1")
nextPermission(t, ctx, phone, "req_2")
// A client that attaches afterwards is replayed both.
writeChat(t, ctx, tablet, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, ctx, tablet, protocol.ChatSubscribe)
first := nextPermission(t, ctx, tablet, "req_1")
second := nextPermission(t, ctx, tablet, "req_2")
if first.Resolved || second.Resolved || first.Action != "bash" || second.Action != "write" {
t.Fatalf("replayed permissions = %+v / %+v, want both still-pending requests", first, second)
}
}
// TestChatResyncAfterReconnect is the resync acceptance test: a client that drops
// and reconnects re-syncs the whole conversation from retained state, including
// the user turn and who sent it.
func TestChatResyncAfterReconnect(t *testing.T) {
broker := scriptedChat()
ctx, _, srv, conns := dialChatServer(t, Options{Chat: broker}, 1)
phone := conns[0]
writeChat(t, ctx, phone, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi", Client: "Pixel 9"}})
readUntil(t, ctx, phone, protocol.ChatPrompt)
readChatTurn(t, ctx, phone)
// The phone's socket drops.
phone.CloseNow()
// A fresh connection re-syncs from retained state.
reconnected := dialChatConn(t, ctx, srv)
writeChat(t, ctx, reconnected, protocol.Message{Type: protocol.ChatMessages, ID: "m1", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}})
response := readUntil(t, ctx, reconnected, protocol.ChatMessages)
if len(response.ChatMessages) != 2 {
t.Fatalf("resync returned %d messages, want user+assistant: %+v", len(response.ChatMessages), response.ChatMessages)
}
user := response.ChatMessages[0]
if user.Role != protocol.ChatRoleUser || user.Client != "Pixel 9" || len(user.Parts) != 1 || user.Parts[0].Text != "hi" {
t.Fatalf("resynced user turn = %+v", user)
}
assistant := response.ChatMessages[1]
if assistant.Role != protocol.ChatRoleAssistant || assistant.ID != "msg_1" {
t.Fatalf("resynced assistant turn = %+v", assistant)
}
}