Files
pipelines/internal/server/chat_unattended_test.go

354 lines
14 KiB
Go

package server
import (
"context"
"path/filepath"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// observedChat is a channelChat whose stream can be opened more than once and
// reports how many streams are live, so a test can prove when the hub observes
// a session with no client attached and when it stops. probe answers the
// server-start ProbeTurn.
type observedChat struct {
*channelChat
smu sync.Mutex
active int
opened int
probe map[string]bool
probes int
}
func newObservedChat() *observedChat {
return &observedChat{channelChat: newChannelChat(), probe: map[string]bool{}}
}
func (c *observedChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
c.smu.Lock()
c.active++
c.opened++
c.smu.Unlock()
defer func() {
c.smu.Lock()
c.active--
c.smu.Unlock()
}()
if ready != nil {
ready()
}
for {
select {
case <-ctx.Done():
return ctx.Err()
case frame := <-c.frames:
if err := emit(frame); err != nil {
return err
}
}
}
}
func (c *observedChat) ProbeTurn(_ context.Context, sessionID string) (bool, error) {
c.smu.Lock()
defer c.smu.Unlock()
c.probes++
return c.probe[sessionID], nil
}
func (c *observedChat) streams() (active, opened int) {
c.smu.Lock()
defer c.smu.Unlock()
return c.active, c.opened
}
func fastUnattended(t *testing.T) {
t.Helper()
check, settle := chatUnattendedCheck, chatUnattendedRecoverSettle
chatUnattendedCheck = 10 * time.Millisecond
chatUnattendedRecoverSettle = 200 * time.Millisecond
t.Cleanup(func() { chatUnattendedCheck, chatUnattendedRecoverSettle = check, settle })
}
func unattendedStore(t *testing.T, replies bool) *pipeline.Store {
t.Helper()
store := openQueueStore(t, filepath.Join(t.TempDir(), "chat.db"))
if err := store.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
if replies {
if _, err := store.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
t.Fatal(err)
}
}
return store
}
// chatNotes returns the recorded app notifications of one kind.
func chatNotes(t *testing.T, store *pipeline.Store, kind string) []protocol.AppNotification {
t.Helper()
active, _, err := store.ActiveAppNotifications(context.Background(), "", time.Now())
if err != nil {
t.Fatal(err)
}
var out []protocol.AppNotification
for _, n := range active {
if n.Kind == kind {
out = append(out, n)
}
}
return out
}
func stepFrameFor(status, turn string) protocol.ChatEventFrame {
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: status, TurnID: turn}}
}
func textFrame(text string) protocol.ChatEventFrame {
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventText, MessageID: "msg_a", Part: &protocol.ChatPart{ID: "msg_a_p"}, Delta: text}
}
// A turn started from the app keeps being observed after the app closes, so
// its finished reply is recorded (app stream) and published (ntfy) once, and
// the observer is torn down when the turn ends.
func TestChatReplyNotifiesAfterLastClientLeaves(t *testing.T) {
fastUnattended(t)
captured := newNtfyCapture(t)
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
waitFor(t, "prompt", func() bool { return broker.promptCount() == 1 })
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
broker.push(textFrame("The build is green."))
conn.CloseNow()
time.Sleep(100 * time.Millisecond) // let the server notice the disconnect
if active, _ := broker.streams(); active != 1 {
t.Fatalf("in-flight turn not observed after the client left: %d streams", active)
}
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
waitFor(t, "ntfy publish", func() bool { return len(captured()) == 1 })
time.Sleep(50 * time.Millisecond)
notes := chatNotes(t, store, protocol.NotificationChatReply)
if got := captured(); len(notes) != 1 || len(got) != 1 || got[0].Title != "Release prep replied" || got[0].Message != "The build is green." {
t.Fatalf("notes=%+v ntfy=%+v", notes, got)
}
if notes[0].Message != "The build is green." || notes[0].Click != "pipelines://chat/ses_1" {
t.Fatalf("note = %+v", notes[0])
}
}
// The chat_reply_finished setting still decides: off (the default) records
// nothing, even though the unattended turn is observed.
func TestChatUnattendedReplyRespectsSetting(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, false)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
time.Sleep(100 * time.Millisecond)
if notes := chatNotes(t, store, protocol.NotificationChatReply); len(notes) != 0 {
t.Fatalf("reply recorded with chat_reply_finished off: %+v", notes)
}
}
// A question the agent asks while no client is attached notifies.
func TestChatQuestionNotifiesWithNoClient(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, false)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "ship?", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Which branch should ship?"}
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
if notes := chatNotes(t, store, protocol.NotificationChatQuestion); notes[0].Message != "Which branch should ship?" {
t.Fatalf("note = %+v", notes[0])
}
// The turn ends with the question withdrawn: nothing is left to observe.
form.Cancelled = true
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A client reattaching mid-turn joins the observing room: the reply is still
// announced exactly once and the client sees the turn end.
func TestChatReattachMidTurnDoesNotDuplicate(t *testing.T) {
fastUnattended(t)
captured := newNtfyCapture(t)
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
broker.push(textFrame("Done."))
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Continue?"}
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
again := dialChatConn(t, server.ctx, server.srv)
writeChat(t, server.ctx, again, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, server.ctx, again, protocol.ChatSubscribe)
if _, opened := broker.streams(); opened != 1 {
t.Fatalf("reattach opened a second stream: %d", opened)
}
form.Resolved = true
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
for {
msg := readUntil(t, server.ctx, again, protocol.ChatEvent)
if e := msg.ChatEvent; e.Kind == protocol.ChatEventStep && e.Step != nil && e.Step.Status == protocol.ChatStepEnded {
break
}
}
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
time.Sleep(100 * time.Millisecond)
if got := captured(); len(got) != 2 {
t.Fatalf("ntfy deliveries = %+v, want one question and one reply", got)
}
if n := len(chatNotes(t, store, protocol.NotificationChatQuestion)) + len(chatNotes(t, store, protocol.NotificationChatReply)); n > 2 {
t.Fatalf("duplicate notifications: %d", n)
}
// The attended room is idle now; the client leaving releases it.
again.CloseNow()
waitFor(t, "idle room released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// An idle session is not observed once its last client leaves, and a server
// start does not observe sessions whose harness reports no running turn.
func TestChatIdleSessionWithNoClientIsNotObserved(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
waitFor(t, "start-up probe", func() bool {
broker.smu.Lock()
defer broker.smu.Unlock()
return broker.probes == 1
})
if _, opened := broker.streams(); opened != 0 {
t.Fatalf("idle session observed at start: %d streams", opened)
}
conn := dialChatConn(t, server.ctx, server.srv)
writeChat(t, server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, server.ctx, conn, protocol.ChatSubscribe)
waitFor(t, "stream", func() bool { active, _ := broker.streams(); return active == 1 })
conn.CloseNow()
waitFor(t, "idle stream released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A turn a previous server process left running is observed again at start,
// with no client, so its reply still notifies; then the observer goes away.
func TestChatRunningTurnObservedAfterRestart(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
broker.probe["ses_1"] = true
startQueueServer(t, broker, store)
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
// The harness re-announces the running turn on attach, then finishes it.
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_9"))
broker.push(textFrame("Finished after the restart."))
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_9"))
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
}
// Authoritative busy survives the settle window even without a replayed start;
// a later authoritative idle result releases it.
func TestChatRecoveredBusySessionWaitsForAuthoritativeIdle(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
broker.probe["ses_1"] = true
startQueueServer(t, broker, store)
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
time.Sleep(2 * chatUnattendedRecoverSettle)
if active, _ := broker.streams(); active != 1 {
t.Fatal("authoritative busy session lost observer")
}
broker.smu.Lock()
broker.probe["ses_1"] = false
broker.smu.Unlock()
waitFor(t, "released after authoritative idle", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A turn that never reports its end does not hold an observer forever.
func TestChatStuckUnattendedTurnIsReleased(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
limit := chatUnattendedTurnLimit
chatUnattendedTurnLimit = 300 * time.Millisecond
t.Cleanup(func() { chatUnattendedTurnLimit = limit })
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
if active, _ := broker.streams(); active != 1 {
t.Fatalf("in-flight turn not observed: %d", active)
}
waitFor(t, "stuck observer released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// At most chatUnattendedLimit chats are kept observing with no client.
func TestChatUnattendedRoomsAreBounded(t *testing.T) {
fastUnattended(t)
h := newChatHub()
defer func() {
h.mu.Lock()
for _, room := range h.rooms {
room.cancel()
}
h.mu.Unlock()
}()
kept := 0
for i := 0; i < chatUnattendedLimit+3; i++ {
id := "ses_" + string(rune('a'+i))
h.mu.Lock()
room := h.roomLocked(id)
room.running = true
sub := &chatSub{hub: h, room: room, clientID: "c", ch: make(chan protocol.Message, 1), done: make(chan struct{})}
room.subs["c"] = sub
h.mu.Unlock()
h.detach(id, sub)
h.mu.Lock()
if h.rooms[id] == room {
kept++
}
h.mu.Unlock()
}
if kept != chatUnattendedLimit {
t.Fatalf("kept %d unattended rooms, want %d", kept, chatUnattendedLimit)
}
}