1b98877d88
chat_sessions rows carry activity (turn_running, background_tasks, seq) from the hub's own state, and chat_activity is pushed to every connection when a session's turn starts or ends or its running background-task count changes. A room whose last viewer left stays observed while background tasks run, bounded by the unattended limits; a Claude Code process exit ends its tasks.
282 lines
8.7 KiB
Go
282 lines
8.7 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
)
|
|
|
|
type shellBarrierBroker struct {
|
|
*fakeChat
|
|
read chan struct{}
|
|
apply chan struct{}
|
|
events chan shellBarrierEvent
|
|
stopped chan struct{}
|
|
reads atomic.Int32
|
|
snapshot []protocol.ChatShell
|
|
}
|
|
|
|
// Exercise the real websocket handler on a loopback scratch server and a fresh
|
|
// SQLite database. A marker on the shared stream bounds the replay assertion
|
|
// without sleeps, polling or a negative-read timeout.
|
|
func TestChatShellReconciliationScratchServer(t *testing.T) {
|
|
b := newShellBarrierBroker()
|
|
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t)))
|
|
defer srv.Close()
|
|
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.CloseNow()
|
|
subscribe := func(id string) {
|
|
t.Helper()
|
|
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
read := func() protocol.Message {
|
|
t.Helper()
|
|
var m protocol.Message
|
|
if err := readTestFrame(ctx, conn, &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return m
|
|
}
|
|
if m := read(); m.Type != protocol.PipelineList {
|
|
t.Fatalf("server greeting = %s", m.Type)
|
|
}
|
|
subscribe("first")
|
|
if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe {
|
|
t.Fatalf("ack = %+v", m)
|
|
}
|
|
shellBarrierWait(t, b.read)
|
|
done := make(chan struct{})
|
|
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done}
|
|
shellBarrierWait(t, done)
|
|
if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited {
|
|
t.Fatalf("exit not visible: %+v", m)
|
|
}
|
|
close(b.apply)
|
|
subscribe("second")
|
|
for {
|
|
m := read()
|
|
if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
|
t.Fatalf("stale replay: %+v", m)
|
|
}
|
|
if m.ID == "second" {
|
|
break
|
|
}
|
|
}
|
|
done = make(chan struct{})
|
|
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done}
|
|
shellBarrierWait(t, done)
|
|
for {
|
|
m := read()
|
|
if m.ChatEvent == nil {
|
|
continue
|
|
}
|
|
if m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
|
t.Fatalf("ghost visible after apply: %+v", m)
|
|
}
|
|
if m.ChatEvent.Shell.ID == "sh_marker" {
|
|
break
|
|
}
|
|
}
|
|
if b.reads.Load() != 1 {
|
|
t.Fatalf("shared observer read count = %d", b.reads.Load())
|
|
}
|
|
conn.CloseNow()
|
|
shellBarrierWait(t, b.stopped)
|
|
}
|
|
|
|
type shellBarrierEvent struct {
|
|
shell protocol.ChatShell
|
|
done chan struct{}
|
|
}
|
|
|
|
func newShellBarrierBroker() *shellBarrierBroker {
|
|
return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})}
|
|
}
|
|
|
|
func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) {
|
|
b.reads.Add(1)
|
|
snapshot := append([]protocol.ChatShell(nil), b.snapshot...)
|
|
b.read <- struct{}{}
|
|
select {
|
|
case <-b.apply:
|
|
return snapshot, nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
|
defer close(b.stopped)
|
|
ready()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case event := <-b.events:
|
|
if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil {
|
|
return err
|
|
}
|
|
close(event.done)
|
|
}
|
|
}
|
|
}
|
|
|
|
func shellBarrierWait(t *testing.T, ch <-chan struct{}) {
|
|
t.Helper()
|
|
select {
|
|
case <-ch:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("barrier timed out")
|
|
}
|
|
}
|
|
|
|
func TestChatShellReconciliationBarrier(t *testing.T) {
|
|
for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} {
|
|
t.Run(status, func(t *testing.T) {
|
|
b := newShellBarrierBroker()
|
|
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
|
h := newChatHub()
|
|
messages := make(chan protocol.Message, 32)
|
|
sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil })
|
|
t.Cleanup(sub.close)
|
|
result := make(chan error, 2)
|
|
go func() { result <- h.ensureObserving(sub.room, b) }()
|
|
shellBarrierWait(t, b.read)
|
|
// A second observer must share the in-flight read, not apply an
|
|
// independently completing older/newer snapshot out of order.
|
|
go func() { result <- h.ensureObserving(sub.room, b) }()
|
|
shell := protocol.ChatShell{ID: "sh_old", Status: status}
|
|
if status == protocol.ChatShellRunning {
|
|
shell.ID = "sh_new"
|
|
shell.Command = "sleep 10"
|
|
}
|
|
done := make(chan struct{})
|
|
select {
|
|
case b.events <- shellBarrierEvent{shell, done}:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("event blocked behind readback")
|
|
}
|
|
shellBarrierWait(t, done)
|
|
select {
|
|
case m := <-messages:
|
|
if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status {
|
|
t.Fatalf("live shell = %+v", m)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("live event not visible during read")
|
|
}
|
|
close(b.apply)
|
|
for range 2 {
|
|
select {
|
|
case err := <-result:
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("observer stuck")
|
|
}
|
|
}
|
|
if b.reads.Load() != 1 {
|
|
t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load())
|
|
}
|
|
snapshot := h.shellSnapshot("ses_1")
|
|
if status == protocol.ChatShellRunning {
|
|
if len(snapshot) != 2 {
|
|
t.Fatalf("new create lost: %+v", snapshot)
|
|
}
|
|
} else if len(snapshot) != 0 {
|
|
t.Fatalf("terminal shell resurrected: %+v", snapshot)
|
|
}
|
|
// Assert the same retained state is replayed to the client.
|
|
h.replayShells(sub)
|
|
for range len(snapshot) {
|
|
select {
|
|
case m := <-messages:
|
|
if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning {
|
|
t.Fatalf("replay = %+v", m)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("replay missing")
|
|
}
|
|
}
|
|
sub.close()
|
|
if status == protocol.ChatShellRunning {
|
|
// Running background tasks keep the room observing with no
|
|
// client (chat_unattended.go); release it to end the stream.
|
|
releaseKeptRoom(t, h, sub.room)
|
|
}
|
|
shellBarrierWait(t, b.stopped)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) {
|
|
b := newShellBarrierBroker()
|
|
h := newChatHub()
|
|
sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil })
|
|
results := make(chan error, 2)
|
|
go func() { results <- h.ensureObserving(sub.room, b) }()
|
|
shellBarrierWait(t, b.read)
|
|
go func() { results <- h.ensureObserving(sub.room, b) }()
|
|
sub.close()
|
|
for range 2 {
|
|
select {
|
|
case err := <-results:
|
|
if err == nil {
|
|
t.Fatal("canceled observer succeeded")
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("canceled waiter stuck")
|
|
}
|
|
}
|
|
shellBarrierWait(t, b.stopped)
|
|
next := newShellBarrierBroker()
|
|
next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}}
|
|
close(next.apply)
|
|
newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil })
|
|
t.Cleanup(newSub.close)
|
|
if err := h.ensureObserving(newSub.room, next); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A late callback from the canceled stream cannot poison the new room,
|
|
// nor can a foreign-session callback from the current stream.
|
|
h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}})
|
|
h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}})
|
|
got := h.shellSnapshot("ses_1")
|
|
if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 {
|
|
t.Fatalf("reconnect state = %+v", got)
|
|
}
|
|
newSub.close()
|
|
releaseKeptRoom(t, h, newSub.room)
|
|
shellBarrierWait(t, next.stopped)
|
|
}
|
|
|
|
// releaseKeptRoom asserts room is still kept observing after its last client
|
|
// left (it holds running background tasks) and releases it.
|
|
func releaseKeptRoom(t *testing.T, h *chatHub, room *chatRoom) {
|
|
t.Helper()
|
|
h.mu.Lock()
|
|
kept := h.rooms[room.sessionID] == room && room.watching
|
|
room.watching = false
|
|
h.releaseRoomLocked(room)
|
|
h.mu.Unlock()
|
|
if !kept {
|
|
t.Fatal("a room with running background tasks was not kept observing")
|
|
}
|
|
}
|