Files
pipelines/internal/server/chat_shell_reconcile_test.go
alan 1b98877d88 Chat list activity: turn running and background tasks on every row, pushed to every connection
chat_sessions rows carry activity (turn_running, background_tasks, seq) from
the hub's own state, and chat_activity is pushed to every connection when a
session's turn starts or ends or its running background-task count changes.
A room whose last viewer left stays observed while background tasks run,
bounded by the unattended limits; a Claude Code process exit ends its tasks.
2026-11-03 19:45:43 +00:00

282 lines
8.7 KiB
Go

package server
import (
"context"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
type shellBarrierBroker struct {
*fakeChat
read chan struct{}
apply chan struct{}
events chan shellBarrierEvent
stopped chan struct{}
reads atomic.Int32
snapshot []protocol.ChatShell
}
// Exercise the real websocket handler on a loopback scratch server and a fresh
// SQLite database. A marker on the shared stream bounds the replay assertion
// without sleeps, polling or a negative-read timeout.
func TestChatShellReconciliationScratchServer(t *testing.T) {
b := newShellBarrierBroker()
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t)))
defer srv.Close()
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
defer conn.CloseNow()
subscribe := func(id string) {
t.Helper()
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
t.Fatal(err)
}
}
read := func() protocol.Message {
t.Helper()
var m protocol.Message
if err := readTestFrame(ctx, conn, &m); err != nil {
t.Fatal(err)
}
return m
}
if m := read(); m.Type != protocol.PipelineList {
t.Fatalf("server greeting = %s", m.Type)
}
subscribe("first")
if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe {
t.Fatalf("ack = %+v", m)
}
shellBarrierWait(t, b.read)
done := make(chan struct{})
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done}
shellBarrierWait(t, done)
if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited {
t.Fatalf("exit not visible: %+v", m)
}
close(b.apply)
subscribe("second")
for {
m := read()
if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
t.Fatalf("stale replay: %+v", m)
}
if m.ID == "second" {
break
}
}
done = make(chan struct{})
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done}
shellBarrierWait(t, done)
for {
m := read()
if m.ChatEvent == nil {
continue
}
if m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
t.Fatalf("ghost visible after apply: %+v", m)
}
if m.ChatEvent.Shell.ID == "sh_marker" {
break
}
}
if b.reads.Load() != 1 {
t.Fatalf("shared observer read count = %d", b.reads.Load())
}
conn.CloseNow()
shellBarrierWait(t, b.stopped)
}
type shellBarrierEvent struct {
shell protocol.ChatShell
done chan struct{}
}
func newShellBarrierBroker() *shellBarrierBroker {
return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})}
}
func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) {
b.reads.Add(1)
snapshot := append([]protocol.ChatShell(nil), b.snapshot...)
b.read <- struct{}{}
select {
case <-b.apply:
return snapshot, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
defer close(b.stopped)
ready()
for {
select {
case <-ctx.Done():
return ctx.Err()
case event := <-b.events:
if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil {
return err
}
close(event.done)
}
}
}
func shellBarrierWait(t *testing.T, ch <-chan struct{}) {
t.Helper()
select {
case <-ch:
case <-time.After(3 * time.Second):
t.Fatal("barrier timed out")
}
}
func TestChatShellReconciliationBarrier(t *testing.T) {
for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} {
t.Run(status, func(t *testing.T) {
b := newShellBarrierBroker()
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
h := newChatHub()
messages := make(chan protocol.Message, 32)
sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil })
t.Cleanup(sub.close)
result := make(chan error, 2)
go func() { result <- h.ensureObserving(sub.room, b) }()
shellBarrierWait(t, b.read)
// A second observer must share the in-flight read, not apply an
// independently completing older/newer snapshot out of order.
go func() { result <- h.ensureObserving(sub.room, b) }()
shell := protocol.ChatShell{ID: "sh_old", Status: status}
if status == protocol.ChatShellRunning {
shell.ID = "sh_new"
shell.Command = "sleep 10"
}
done := make(chan struct{})
select {
case b.events <- shellBarrierEvent{shell, done}:
case <-time.After(time.Second):
t.Fatal("event blocked behind readback")
}
shellBarrierWait(t, done)
select {
case m := <-messages:
if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status {
t.Fatalf("live shell = %+v", m)
}
case <-time.After(time.Second):
t.Fatal("live event not visible during read")
}
close(b.apply)
for range 2 {
select {
case err := <-result:
if err != nil {
t.Fatal(err)
}
case <-time.After(time.Second):
t.Fatal("observer stuck")
}
}
if b.reads.Load() != 1 {
t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load())
}
snapshot := h.shellSnapshot("ses_1")
if status == protocol.ChatShellRunning {
if len(snapshot) != 2 {
t.Fatalf("new create lost: %+v", snapshot)
}
} else if len(snapshot) != 0 {
t.Fatalf("terminal shell resurrected: %+v", snapshot)
}
// Assert the same retained state is replayed to the client.
h.replayShells(sub)
for range len(snapshot) {
select {
case m := <-messages:
if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning {
t.Fatalf("replay = %+v", m)
}
case <-time.After(time.Second):
t.Fatal("replay missing")
}
}
sub.close()
if status == protocol.ChatShellRunning {
// Running background tasks keep the room observing with no
// client (chat_unattended.go); release it to end the stream.
releaseKeptRoom(t, h, sub.room)
}
shellBarrierWait(t, b.stopped)
})
}
}
func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) {
b := newShellBarrierBroker()
h := newChatHub()
sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil })
results := make(chan error, 2)
go func() { results <- h.ensureObserving(sub.room, b) }()
shellBarrierWait(t, b.read)
go func() { results <- h.ensureObserving(sub.room, b) }()
sub.close()
for range 2 {
select {
case err := <-results:
if err == nil {
t.Fatal("canceled observer succeeded")
}
case <-time.After(time.Second):
t.Fatal("canceled waiter stuck")
}
}
shellBarrierWait(t, b.stopped)
next := newShellBarrierBroker()
next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}}
close(next.apply)
newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil })
t.Cleanup(newSub.close)
if err := h.ensureObserving(newSub.room, next); err != nil {
t.Fatal(err)
}
// A late callback from the canceled stream cannot poison the new room,
// nor can a foreign-session callback from the current stream.
h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}})
h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}})
got := h.shellSnapshot("ses_1")
if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 {
t.Fatalf("reconnect state = %+v", got)
}
newSub.close()
releaseKeptRoom(t, h, newSub.room)
shellBarrierWait(t, next.stopped)
}
// releaseKeptRoom asserts room is still kept observing after its last client
// left (it holds running background tasks) and releases it.
func releaseKeptRoom(t *testing.T, h *chatHub, room *chatRoom) {
t.Helper()
h.mu.Lock()
kept := h.rooms[room.sessionID] == room && room.watching
room.watching = false
h.releaseRoomLocked(room)
h.mu.Unlock()
if !kept {
t.Fatal("a room with running background tasks was not kept observing")
}
}