354 lines
14 KiB
Go
354 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// observedChat is a channelChat whose stream can be opened more than once and
|
|
// reports how many streams are live, so a test can prove when the hub observes
|
|
// a session with no client attached and when it stops. probe answers the
|
|
// server-start ProbeTurn.
|
|
type observedChat struct {
|
|
*channelChat
|
|
smu sync.Mutex
|
|
active int
|
|
opened int
|
|
probe map[string]bool
|
|
probes int
|
|
}
|
|
|
|
func newObservedChat() *observedChat {
|
|
return &observedChat{channelChat: newChannelChat(), probe: map[string]bool{}}
|
|
}
|
|
|
|
func (c *observedChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
|
c.smu.Lock()
|
|
c.active++
|
|
c.opened++
|
|
c.smu.Unlock()
|
|
defer func() {
|
|
c.smu.Lock()
|
|
c.active--
|
|
c.smu.Unlock()
|
|
}()
|
|
if ready != nil {
|
|
ready()
|
|
}
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case frame := <-c.frames:
|
|
if err := emit(frame); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *observedChat) ProbeTurn(_ context.Context, sessionID string) (bool, error) {
|
|
c.smu.Lock()
|
|
defer c.smu.Unlock()
|
|
c.probes++
|
|
return c.probe[sessionID], nil
|
|
}
|
|
|
|
func (c *observedChat) streams() (active, opened int) {
|
|
c.smu.Lock()
|
|
defer c.smu.Unlock()
|
|
return c.active, c.opened
|
|
}
|
|
|
|
func fastUnattended(t *testing.T) {
|
|
t.Helper()
|
|
check, settle := chatUnattendedCheck, chatUnattendedRecoverSettle
|
|
chatUnattendedCheck = 10 * time.Millisecond
|
|
chatUnattendedRecoverSettle = 200 * time.Millisecond
|
|
t.Cleanup(func() { chatUnattendedCheck, chatUnattendedRecoverSettle = check, settle })
|
|
}
|
|
|
|
func unattendedStore(t *testing.T, replies bool) *pipeline.Store {
|
|
t.Helper()
|
|
store := openQueueStore(t, filepath.Join(t.TempDir(), "chat.db"))
|
|
if err := store.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if replies {
|
|
if _, err := store.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return store
|
|
}
|
|
|
|
// chatNotes returns the recorded app notifications of one kind.
|
|
func chatNotes(t *testing.T, store *pipeline.Store, kind string) []protocol.AppNotification {
|
|
t.Helper()
|
|
active, _, err := store.ActiveAppNotifications(context.Background(), "", time.Now())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out []protocol.AppNotification
|
|
for _, n := range active {
|
|
if n.Kind == kind {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func stepFrameFor(status, turn string) protocol.ChatEventFrame {
|
|
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: status, TurnID: turn}}
|
|
}
|
|
|
|
func textFrame(text string) protocol.ChatEventFrame {
|
|
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventText, MessageID: "msg_a", Part: &protocol.ChatPart{ID: "msg_a_p"}, Delta: text}
|
|
}
|
|
|
|
// A turn started from the app keeps being observed after the app closes, so
|
|
// its finished reply is recorded (app stream) and published (ntfy) once, and
|
|
// the observer is torn down when the turn ends.
|
|
func TestChatReplyNotifiesAfterLastClientLeaves(t *testing.T) {
|
|
fastUnattended(t)
|
|
captured := newNtfyCapture(t)
|
|
store := unattendedStore(t, true)
|
|
broker := newObservedChat()
|
|
server := startQueueServer(t, broker, store)
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
|
waitFor(t, "prompt", func() bool { return broker.promptCount() == 1 })
|
|
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
|
broker.push(textFrame("The build is green."))
|
|
conn.CloseNow()
|
|
time.Sleep(100 * time.Millisecond) // let the server notice the disconnect
|
|
if active, _ := broker.streams(); active != 1 {
|
|
t.Fatalf("in-flight turn not observed after the client left: %d streams", active)
|
|
}
|
|
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
|
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
|
|
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
|
waitFor(t, "ntfy publish", func() bool { return len(captured()) == 1 })
|
|
time.Sleep(50 * time.Millisecond)
|
|
notes := chatNotes(t, store, protocol.NotificationChatReply)
|
|
if got := captured(); len(notes) != 1 || len(got) != 1 || got[0].Title != "Release prep replied" || got[0].Message != "The build is green." {
|
|
t.Fatalf("notes=%+v ntfy=%+v", notes, got)
|
|
}
|
|
if notes[0].Message != "The build is green." || notes[0].Click != "pipelines://chat/ses_1" {
|
|
t.Fatalf("note = %+v", notes[0])
|
|
}
|
|
}
|
|
|
|
// The chat_reply_finished setting still decides: off (the default) records
|
|
// nothing, even though the unattended turn is observed.
|
|
func TestChatUnattendedReplyRespectsSetting(t *testing.T) {
|
|
fastUnattended(t)
|
|
t.Setenv("PIPELINES_NTFY_URL", "")
|
|
store := unattendedStore(t, false)
|
|
broker := newObservedChat()
|
|
server := startQueueServer(t, broker, store)
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
|
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
|
conn.CloseNow()
|
|
time.Sleep(100 * time.Millisecond)
|
|
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
|
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
|
time.Sleep(100 * time.Millisecond)
|
|
if notes := chatNotes(t, store, protocol.NotificationChatReply); len(notes) != 0 {
|
|
t.Fatalf("reply recorded with chat_reply_finished off: %+v", notes)
|
|
}
|
|
}
|
|
|
|
// A question the agent asks while no client is attached notifies.
|
|
func TestChatQuestionNotifiesWithNoClient(t *testing.T) {
|
|
fastUnattended(t)
|
|
t.Setenv("PIPELINES_NTFY_URL", "")
|
|
store := unattendedStore(t, false)
|
|
broker := newObservedChat()
|
|
server := startQueueServer(t, broker, store)
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
sendPrompt(t, server.ctx, conn, "p1", "ship?", "op-1")
|
|
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
|
conn.CloseNow()
|
|
time.Sleep(100 * time.Millisecond)
|
|
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Which branch should ship?"}
|
|
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
|
|
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
|
|
if notes := chatNotes(t, store, protocol.NotificationChatQuestion); notes[0].Message != "Which branch should ship?" {
|
|
t.Fatalf("note = %+v", notes[0])
|
|
}
|
|
// The turn ends with the question withdrawn: nothing is left to observe.
|
|
form.Cancelled = true
|
|
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
|
|
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
|
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
|
}
|
|
|
|
// A client reattaching mid-turn joins the observing room: the reply is still
|
|
// announced exactly once and the client sees the turn end.
|
|
func TestChatReattachMidTurnDoesNotDuplicate(t *testing.T) {
|
|
fastUnattended(t)
|
|
captured := newNtfyCapture(t)
|
|
store := unattendedStore(t, true)
|
|
broker := newObservedChat()
|
|
server := startQueueServer(t, broker, store)
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
|
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
|
broker.push(textFrame("Done."))
|
|
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Continue?"}
|
|
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
|
|
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
|
|
conn.CloseNow()
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
again := dialChatConn(t, server.ctx, server.srv)
|
|
writeChat(t, server.ctx, again, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
|
|
readUntil(t, server.ctx, again, protocol.ChatSubscribe)
|
|
if _, opened := broker.streams(); opened != 1 {
|
|
t.Fatalf("reattach opened a second stream: %d", opened)
|
|
}
|
|
form.Resolved = true
|
|
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
|
|
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
|
for {
|
|
msg := readUntil(t, server.ctx, again, protocol.ChatEvent)
|
|
if e := msg.ChatEvent; e.Kind == protocol.ChatEventStep && e.Step != nil && e.Step.Status == protocol.ChatStepEnded {
|
|
break
|
|
}
|
|
}
|
|
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
|
|
time.Sleep(100 * time.Millisecond)
|
|
if got := captured(); len(got) != 2 {
|
|
t.Fatalf("ntfy deliveries = %+v, want one question and one reply", got)
|
|
}
|
|
if n := len(chatNotes(t, store, protocol.NotificationChatQuestion)) + len(chatNotes(t, store, protocol.NotificationChatReply)); n > 2 {
|
|
t.Fatalf("duplicate notifications: %d", n)
|
|
}
|
|
// The attended room is idle now; the client leaving releases it.
|
|
again.CloseNow()
|
|
waitFor(t, "idle room released", func() bool { active, _ := broker.streams(); return active == 0 })
|
|
}
|
|
|
|
// An idle session is not observed once its last client leaves, and a server
|
|
// start does not observe sessions whose harness reports no running turn.
|
|
func TestChatIdleSessionWithNoClientIsNotObserved(t *testing.T) {
|
|
fastUnattended(t)
|
|
t.Setenv("PIPELINES_NTFY_URL", "")
|
|
store := unattendedStore(t, true)
|
|
broker := newObservedChat()
|
|
server := startQueueServer(t, broker, store)
|
|
waitFor(t, "start-up probe", func() bool {
|
|
broker.smu.Lock()
|
|
defer broker.smu.Unlock()
|
|
return broker.probes == 1
|
|
})
|
|
if _, opened := broker.streams(); opened != 0 {
|
|
t.Fatalf("idle session observed at start: %d streams", opened)
|
|
}
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
writeChat(t, server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
|
|
readUntil(t, server.ctx, conn, protocol.ChatSubscribe)
|
|
waitFor(t, "stream", func() bool { active, _ := broker.streams(); return active == 1 })
|
|
conn.CloseNow()
|
|
waitFor(t, "idle stream released", func() bool { active, _ := broker.streams(); return active == 0 })
|
|
}
|
|
|
|
// A turn a previous server process left running is observed again at start,
|
|
// with no client, so its reply still notifies; then the observer goes away.
|
|
func TestChatRunningTurnObservedAfterRestart(t *testing.T) {
|
|
fastUnattended(t)
|
|
t.Setenv("PIPELINES_NTFY_URL", "")
|
|
store := unattendedStore(t, true)
|
|
broker := newObservedChat()
|
|
broker.probe["ses_1"] = true
|
|
startQueueServer(t, broker, store)
|
|
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
|
|
// The harness re-announces the running turn on attach, then finishes it.
|
|
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_9"))
|
|
broker.push(textFrame("Finished after the restart."))
|
|
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_9"))
|
|
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
|
|
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
|
}
|
|
|
|
// Authoritative busy survives the settle window even without a replayed start;
|
|
// a later authoritative idle result releases it.
|
|
func TestChatRecoveredBusySessionWaitsForAuthoritativeIdle(t *testing.T) {
|
|
fastUnattended(t)
|
|
t.Setenv("PIPELINES_NTFY_URL", "")
|
|
store := unattendedStore(t, true)
|
|
broker := newObservedChat()
|
|
broker.probe["ses_1"] = true
|
|
startQueueServer(t, broker, store)
|
|
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
|
|
time.Sleep(2 * chatUnattendedRecoverSettle)
|
|
if active, _ := broker.streams(); active != 1 {
|
|
t.Fatal("authoritative busy session lost observer")
|
|
}
|
|
broker.smu.Lock()
|
|
broker.probe["ses_1"] = false
|
|
broker.smu.Unlock()
|
|
waitFor(t, "released after authoritative idle", func() bool { active, _ := broker.streams(); return active == 0 })
|
|
}
|
|
|
|
// A turn that never reports its end does not hold an observer forever.
|
|
func TestChatStuckUnattendedTurnIsReleased(t *testing.T) {
|
|
fastUnattended(t)
|
|
t.Setenv("PIPELINES_NTFY_URL", "")
|
|
limit := chatUnattendedTurnLimit
|
|
chatUnattendedTurnLimit = 300 * time.Millisecond
|
|
t.Cleanup(func() { chatUnattendedTurnLimit = limit })
|
|
store := unattendedStore(t, true)
|
|
broker := newObservedChat()
|
|
server := startQueueServer(t, broker, store)
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
|
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
|
conn.CloseNow()
|
|
time.Sleep(100 * time.Millisecond)
|
|
if active, _ := broker.streams(); active != 1 {
|
|
t.Fatalf("in-flight turn not observed: %d", active)
|
|
}
|
|
waitFor(t, "stuck observer released", func() bool { active, _ := broker.streams(); return active == 0 })
|
|
}
|
|
|
|
// At most chatUnattendedLimit chats are kept observing with no client.
|
|
func TestChatUnattendedRoomsAreBounded(t *testing.T) {
|
|
fastUnattended(t)
|
|
h := newChatHub()
|
|
defer func() {
|
|
h.mu.Lock()
|
|
for _, room := range h.rooms {
|
|
room.cancel()
|
|
}
|
|
h.mu.Unlock()
|
|
}()
|
|
kept := 0
|
|
for i := 0; i < chatUnattendedLimit+3; i++ {
|
|
id := "ses_" + string(rune('a'+i))
|
|
h.mu.Lock()
|
|
room := h.roomLocked(id)
|
|
room.running = true
|
|
sub := &chatSub{hub: h, room: room, clientID: "c", ch: make(chan protocol.Message, 1), done: make(chan struct{})}
|
|
room.subs["c"] = sub
|
|
h.mu.Unlock()
|
|
h.detach(id, sub)
|
|
h.mu.Lock()
|
|
if h.rooms[id] == room {
|
|
kept++
|
|
}
|
|
h.mu.Unlock()
|
|
}
|
|
if kept != chatUnattendedLimit {
|
|
t.Fatalf("kept %d unattended rooms, want %d", kept, chatUnattendedLimit)
|
|
}
|
|
}
|