Keep observing chats with an in-flight turn after the last client leaves

Chat attention notifications (questions, approvals, finished replies) come
from a session's event stream, which the hub only read while a room existed;
detach destroyed the room when the last client left, so a turn that finished
or asked with the app closed notified nobody.

A room whose last client leaves now stays observing, unattended, while its
turn is in flight or it holds an unanswered question/approval, and is torn
down when the turn ends (or after 6h for a stuck turn); at most 16 such rooms.
At server start, recently used open sessions whose harness reports a running
turn (OpenCode /api/session/active, Codex thread/turns/list) are observed
again so their reply still notifies. Settings, dedupe and app_notifications
recording are unchanged.
This commit is contained in:
2026-11-02 16:07:42 +00:00
parent 467f3eafa7
commit 80022ea41a
13 changed files with 820 additions and 25 deletions
+8
View File
@@ -100,6 +100,14 @@ func TestCompactionTransportLossAndInterruptClearDoNotReleaseTurn(t *testing.T)
room.cancel()
h.drainStream(room, result)
}
if mode == "detach" {
// The last client leaving a running turn keeps the room observing
// unattended: the turn, and its live compaction, are still current.
if h.rooms[room.sessionID] != room || !room.watching || !room.running {
t.Fatal("detach released the room of a live turn")
}
return
}
if h.turns[room.sessionID].compaction != nil || !room.running {
t.Fatal("cleanup stuck label or released live turn")
}
+4 -1
View File
@@ -58,6 +58,9 @@ type chatHub struct {
// operation ids, so a client retry is acknowledged instead of re-sent.
recentOps map[string]struct{}
recentOpOrder []string
// unattended holds the unattended-observation timings (chat_unattended.go),
// fixed when the hub is built.
unattended chatUnattendedTimings
}
// chatPart is the retained form of one part. Text and reasoning deltas append to
@@ -121,7 +124,7 @@ type chatResolution struct {
}
func newChatHub() *chatHub {
return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now}
return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now, unattended: currentChatUnattendedTimings()}
}
// nextUserMessageID mints a stable id for one echoed user turn.
+16
View File
@@ -701,3 +701,19 @@ func (m *chatManager) TurnRunning(sessionID string) bool {
reporter, ok := adapter.(chatTurnReporter)
return ok && reporter.TurnRunning(sessionID)
}
// ProbeTurn asks the session's own harness whether a turn is in flight. It is
// the server-start probe (chat_unattended.go): a harness that can be asked
// (OpenCode, Codex) is asked; otherwise its in-memory state is used, which for
// Claude Code (a child of this server) is empty after a restart.
func (m *chatManager) ProbeTurn(ctx context.Context, sessionID string) (bool, error) {
adapter, err := m.brokerFor(ctx, sessionID)
if err != nil {
return false, err
}
if prober, ok := adapter.(chatTurnProber); ok {
return prober.ProbeTurn(ctx, sessionID)
}
reporter, ok := adapter.(chatTurnReporter)
return ok && reporter.TurnRunning(sessionID), nil
}
+34 -12
View File
@@ -167,6 +167,14 @@ type chatRoom struct {
// notBefore holds back a recovered queue until the restarted stream had a
// chance to report a turn the harness is still running.
notBefore time.Time
// watching is set while the unattended watcher (chat_unattended.go) owns
// this room's lifetime: the room has no client but is kept observing so a
// finished turn or a question still notifies. unattendedSince bounds how
// long a stuck turn may hold it; holdUntil keeps a room recovered at start
// alive until its stream has had a chance to report the running turn.
watching bool
unattendedSince time.Time
holdUntil time.Time
}
func newChatRoom(base context.Context, sessionID string) *chatRoom {
@@ -309,11 +317,13 @@ func (h *chatHub) attach(sessionID, clientID, label string, broker chatBroker, w
}
// detach drops one subscription. The room (and its broker stream and queue) is
// released once its last client leaves and it has no prompt left to deliver,
// so a session with no watchers holds no goroutine. A room holding queued
// released once its last client leaves and it has nothing left to do, so an
// idle session with no watchers holds no goroutine. A room holding queued
// prompts stays (with its stream) until its runner has delivered them: the
// queue belongs to the session, not to the clients watching it. A running turn
// with nothing queued behind it needs no room; the harness finishes it alone.
// queue belongs to the session, not to the clients watching it. A room whose
// turn is still in flight, or which holds an unanswered question or approval,
// also stays (bounded, see chat_unattended.go) so the operator is still
// notified when the turn finishes or the agent asks while no app is open.
func (h *chatHub) detach(sessionID string, sub *chatSub) {
h.mu.Lock()
room := h.rooms[sessionID]
@@ -322,8 +332,8 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) {
return
}
delete(room.subs, sub.clientID)
if len(room.subs) == 0 && len(room.queue) == 0 {
h.releaseRoomLocked(room)
if len(room.subs) == 0 {
h.leaveUnattendedLocked(room)
}
h.mu.Unlock()
}
@@ -461,6 +471,11 @@ func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
}
if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded && !frame.Step.Intermediate {
h.finishTurnLocked(room)
// An unwatched room kept only for this turn is done once it ends (its
// reply attention was noted above).
if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) {
h.releaseRoomLocked(room)
}
}
// Shell replay and live lifecycle delivery must enqueue in the same order
// as folding. Otherwise an exit can overtake an older running replay.
@@ -561,10 +576,12 @@ func (h *chatHub) finishTurnLocked(room *chatRoom) {
// turnInFlight reports whether sessionID has a turn in flight, so chat_close
// knows whether to stop one through the broker interrupt before it records the
// closed state. A room exists only while a client is watching the session, and
// running is set by the turn runner for the whole turn (a prompt queued behind a
// closed state. A room exists only while a client is watching the session or
// it still has work, and running is set by the turn runner for the whole turn (a prompt queued behind a
// live turn leaves it set), so an idle or unwatched session reports false and no
// interrupt is sent. It is one read under the hub mutex and mutates nothing.
// interrupt is sent. (A room also outlives its last client while its turn is in
// flight, see chat_unattended.go.) It is one read under the hub mutex and
// mutates nothing.
func (h *chatHub) turnInFlight(sessionID string) bool {
h.mu.Lock()
defer h.mu.Unlock()
@@ -709,10 +726,15 @@ func (h *chatHub) runRoom(room *chatRoom) {
}
h.runTurn(room, p, done)
// The last client may have left while this turn ran; an unwatched room
// stays only while it still has prompts to deliver.
// stays only while it still has prompts to deliver or something the
// operator must be told about.
h.mu.Lock()
if len(room.subs) == 0 && !room.running && len(room.queue) == 0 {
h.releaseRoomLocked(room)
if len(room.subs) == 0 && len(room.queue) == 0 {
if h.keepUnattendedLocked(room, false, true) {
h.watchUnattendedLocked(room)
} else {
h.releaseRoomLocked(room)
}
}
h.mu.Unlock()
}
+24 -9
View File
@@ -770,12 +770,14 @@ func TestChatMessagesRejectsInvalidSession(t *testing.T) {
}
}
// The stream of a session whose turn is still in flight outlives the client
// (so its reply still notifies), but no goroutine outlives the server.
func TestChatDisconnectCancelsStream(t *testing.T) {
broker := scriptedChat()
broker.block = true
broker.started = make(chan struct{})
broker.done = make(chan struct{})
ctx, _, conn := dialChat(t, broker)
ctx, cancel, conn := dialChat(t, broker)
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: "r1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi"}}); err != nil {
t.Fatal(err)
}
@@ -789,12 +791,17 @@ func TestChatDisconnectCancelsStream(t *testing.T) {
t.Fatal("stream never started")
}
conn.CloseNow()
// The server's read loop returns on disconnect, canceling the connection
// context the subscription observes; no goroutine may keep running.
// The turn is still in flight, so the session stays observed unattended.
select {
case <-broker.done:
t.Fatal("chat stream of an in-flight turn was canceled on disconnect")
case <-time.After(300 * time.Millisecond):
}
cancel()
select {
case <-broker.done:
case <-time.After(3 * time.Second):
t.Fatal("chat stream was not canceled on disconnect")
t.Fatal("chat stream was not canceled on shutdown")
}
if prompts, _, _ := broker.counts(); prompts != 1 {
t.Fatalf("prompt forwarded %d times, want 1", prompts)
@@ -2205,9 +2212,9 @@ func TestChatPermissionDedupesRepeatedPolls(t *testing.T) {
}
}
// TestChatDisconnectCancelsPermissionPoller pins that the permission poller is
// canceled on disconnect (reusing the stream's context), so it cannot leak a
// goroutine after the client goes away.
// TestChatDisconnectCancelsPermissionPoller pins that the permission poller
// shares the room's context: it survives a disconnect while the turn is in
// flight and cannot leak a goroutine past the server.
func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
broker := scriptedChat()
broker.block = true
@@ -2215,7 +2222,7 @@ func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
broker.done = make(chan struct{})
broker.permBlock = true
broker.permDone = make(chan struct{})
ctx, _, conn := dialChat(t, broker)
ctx, cancel, conn := dialChat(t, broker)
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: "r1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi"}}); err != nil {
t.Fatal(err)
}
@@ -2234,10 +2241,18 @@ func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
default:
}
conn.CloseNow()
// An in-flight turn keeps its poller (an approval it raises still
// notifies); shutting the server down cancels it.
select {
case <-broker.permDone:
t.Fatal("permission poller of an in-flight turn was canceled on disconnect")
case <-time.After(300 * time.Millisecond):
}
cancel()
select {
case <-broker.permDone:
case <-time.After(3 * time.Second):
t.Fatal("permission poller was not canceled on disconnect")
t.Fatal("permission poller was not canceled on shutdown")
}
select {
case <-broker.done:
+259
View File
@@ -0,0 +1,259 @@
package server
import (
"context"
"log"
"time"
)
// Chat attention notifications (chat_question, chat_approval, chat_reply) are
// derived from a session's live event stream, and the hub only reads that
// stream while the session has a room. A room normally lives while a client
// watches the chat, but the operator relies on these notifications precisely
// when no app is open. So a room whose last client leaves stays observing,
// unattended, while its turn is in flight or it holds an unanswered question
// or approval, and is torn down once that work is done. The same unattended
// observation is started at server start for sessions whose harness reports a
// running turn, so a reply that finishes after a restart still notifies.
//
// Resources are bounded: at most chatUnattendedLimit rooms are kept without a
// client (rooms holding queued prompts are not counted: they were always kept
// until delivered), a stuck turn is released after chatUnattendedTurnLimit,
// and an idle session is never observed.
// chatUnattendedLimit bounds the rooms kept observing with no client attached
// and no queued prompt.
const chatUnattendedLimit = 16
// chatRecoverSessionLimit bounds how many recently used open sessions are
// probed for a running turn at server start.
const chatRecoverSessionLimit = 32
var (
// chatUnattendedCheck is how often an unattended room re-checks whether it
// still has a reason to observe its session.
chatUnattendedCheck = 5 * time.Second
// chatUnattendedTurnLimit releases an unattended room whose turn never
// reports an end (a stuck harness or a lost end event).
chatUnattendedTurnLimit = 6 * time.Hour
// chatUnattendedRecoverSettle keeps a room recovered at start alive until
// its freshly attached stream has had time to announce the running turn.
chatUnattendedRecoverSettle = 15 * time.Second
// chatRecoverWindow limits start-up probing to sessions used recently.
chatRecoverWindow = 24 * time.Hour
)
// chatUnattendedTimings is the hub's copy of the timings above, taken when the
// hub is built so its goroutines never read the package variables.
type chatUnattendedTimings struct {
check, turnLimit, settle, window time.Duration
}
func currentChatUnattendedTimings() chatUnattendedTimings {
return chatUnattendedTimings{check: chatUnattendedCheck, turnLimit: chatUnattendedTurnLimit, settle: chatUnattendedRecoverSettle, window: chatRecoverWindow}
}
// chatTurnProber is implemented by a broker that can ask the harness itself,
// with I/O, whether a session's turn is in flight. It is used once at server
// start, when the in-memory turn state is empty.
type chatTurnProber interface {
ProbeTurn(ctx context.Context, sessionID string) (bool, error)
}
// keepUnattendedLocked reports whether a room with no client still has work
// the operator must hear about: a turn in flight (the runner's, or one its
// stream announced), an unresolved approval or question, or a harness that
// reports its own turn running (busy, read by the caller off the hub mutex).
// hold honours the start-up settle window. Queued prompts are the caller's
// concern. Callers hold h.mu.
func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool {
if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 {
return true
}
return hold && h.clock().Before(room.holdUntil)
}
// unattendedCountLocked counts the rooms kept observing with no client and no
// queued prompt, other than except. Callers hold h.mu.
func (h *chatHub) unattendedCountLocked(except *chatRoom) int {
n := 0
for _, room := range h.rooms {
if room != except && room.watching && len(room.subs) == 0 && len(room.queue) == 0 {
n++
}
}
return n
}
// leaveUnattendedLocked decides the fate of a room whose last client just
// left: a room holding queued prompts stays for its runner (as before); a room
// with an in-flight turn or pending attention keeps observing unattended,
// within the bound; anything else is released. Callers hold h.mu.
func (h *chatHub) leaveUnattendedLocked(room *chatRoom) {
if len(room.queue) > 0 {
h.watchUnattendedLocked(room)
return
}
if !h.keepUnattendedLocked(room, false, true) {
h.releaseRoomLocked(room)
return
}
if !room.watching && h.unattendedCountLocked(room) >= chatUnattendedLimit {
log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit)
h.releaseRoomLocked(room)
return
}
h.watchUnattendedLocked(room)
}
// watchUnattendedLocked starts the room's unattended watcher once. Callers
// hold h.mu.
func (h *chatHub) watchUnattendedLocked(room *chatRoom) {
if room.watching || room.ctx.Err() != nil {
return
}
room.watching = true
room.unattendedSince = h.clock()
go h.watchUnattended(room)
}
func (h *chatHub) watchUnattended(room *chatRoom) {
check := h.unattended.check
if check <= 0 {
check = 5 * time.Second
}
ticker := time.NewTicker(check)
defer ticker.Stop()
for {
select {
case <-room.ctx.Done():
return
case <-ticker.C:
}
if h.reapUnattended(room) {
return
}
}
}
// reapUnattended re-checks one unattended room and reports whether its
// watcher is finished: a client reattached (the room is attended again), or
// the room had nothing left to observe, or its turn exceeded the limit, and
// was released.
func (h *chatHub) reapUnattended(room *chatRoom) bool {
h.mu.Lock()
if done := h.attendedLocked(room); done {
h.mu.Unlock()
return true
}
broker := room.broker
h.mu.Unlock()
// TurnRunning performs no I/O but may take the broker's own lock, so it is
// never asked under the hub mutex.
busy := brokerTurnRunning(broker, room.sessionID)
h.mu.Lock()
defer h.mu.Unlock()
if done := h.attendedLocked(room); done {
return true
}
if len(room.queue) > 0 {
// The runner owns a room with queued prompts until they are delivered;
// the stuck-turn limit counts from the last of them.
room.unattendedSince = h.clock()
return false
}
expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit
if !expired && h.keepUnattendedLocked(room, busy, true) {
return false
}
if expired {
log.Printf("chat session %s: released the unattended observer after %s without the turn ending", room.sessionID, h.unattended.turnLimit)
}
room.watching = false
h.releaseRoomLocked(room)
return true
}
// attendedLocked reports whether the watcher should stop because the room is
// gone or a client is watching it again. Callers hold h.mu.
func (h *chatHub) attendedLocked(room *chatRoom) bool {
if h.rooms[room.sessionID] != room || room.ctx.Err() != nil {
room.watching = false
return true
}
if len(room.subs) > 0 {
room.watching = false
room.unattendedSince = time.Time{}
return true
}
return false
}
// observeRunningTurns is the start-up half: each recently used open operator
// session whose harness reports a turn in flight gets an unattended room
// observing its stream, so the reply (or a question) still notifies. It is
// best effort per harness: OpenCode and Codex keep running across a Pipelines
// restart and report it; Claude Code runs as a child of the server and so has
// no turn left after a restart.
func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
prober, ok := broker.(chatTurnProber)
if !ok || h.registry == nil {
return
}
sessions, err := h.registry.OperatorSessions(ctx)
if err != nil {
log.Printf("chat turn recovery: %v", err)
return
}
cutoff := h.clock().Add(-h.unattended.window).UnixMilli()
probed := 0
for _, session := range sessions {
if ctx.Err() != nil {
return
}
if session.ClosedAt != 0 || session.UpdatedAt < cutoff || !validChatSessionID(session.ID) {
continue
}
if probed >= chatRecoverSessionLimit {
break
}
probed++
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
running, err := prober.ProbeTurn(probeCtx, session.ID)
cancel()
if err != nil {
log.Printf("chat turn recovery probe session=%s failed (%T)", session.ID, err)
continue
}
if running {
h.observeUnattended(session.ID, broker)
}
}
}
// observeUnattended opens (or reuses) the session's room with no client and
// observes its stream until the turn the harness reported has ended.
func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
h.mu.Lock()
if h.rooms[sessionID] == nil && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
h.mu.Unlock()
log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit)
return
}
room := h.roomLocked(sessionID)
if room.broker == nil {
room.broker = broker
}
if len(room.subs) == 0 {
room.holdUntil = h.clock().Add(h.unattended.settle)
h.watchUnattendedLocked(room)
}
h.mu.Unlock()
if err := h.ensureObserving(room, broker); err != nil {
if room.ctx.Err() == nil {
log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err)
}
return
}
h.startRoomPoller(room, broker)
}
+346
View File
@@ -0,0 +1,346 @@
package server
import (
"context"
"path/filepath"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// observedChat is a channelChat whose stream can be opened more than once and
// reports how many streams are live, so a test can prove when the hub observes
// a session with no client attached and when it stops. probe answers the
// server-start ProbeTurn.
type observedChat struct {
*channelChat
smu sync.Mutex
active int
opened int
probe map[string]bool
probes int
}
func newObservedChat() *observedChat {
return &observedChat{channelChat: newChannelChat(), probe: map[string]bool{}}
}
func (c *observedChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
c.smu.Lock()
c.active++
c.opened++
c.smu.Unlock()
defer func() {
c.smu.Lock()
c.active--
c.smu.Unlock()
}()
if ready != nil {
ready()
}
for {
select {
case <-ctx.Done():
return ctx.Err()
case frame := <-c.frames:
if err := emit(frame); err != nil {
return err
}
}
}
}
func (c *observedChat) ProbeTurn(_ context.Context, sessionID string) (bool, error) {
c.smu.Lock()
defer c.smu.Unlock()
c.probes++
return c.probe[sessionID], nil
}
func (c *observedChat) streams() (active, opened int) {
c.smu.Lock()
defer c.smu.Unlock()
return c.active, c.opened
}
func fastUnattended(t *testing.T) {
t.Helper()
check, settle := chatUnattendedCheck, chatUnattendedRecoverSettle
chatUnattendedCheck = 10 * time.Millisecond
chatUnattendedRecoverSettle = 200 * time.Millisecond
t.Cleanup(func() { chatUnattendedCheck, chatUnattendedRecoverSettle = check, settle })
}
func unattendedStore(t *testing.T, replies bool) *pipeline.Store {
t.Helper()
store := openQueueStore(t, filepath.Join(t.TempDir(), "chat.db"))
if err := store.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
if replies {
if _, err := store.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
t.Fatal(err)
}
}
return store
}
// chatNotes returns the recorded app notifications of one kind.
func chatNotes(t *testing.T, store *pipeline.Store, kind string) []protocol.AppNotification {
t.Helper()
active, _, err := store.ActiveAppNotifications(context.Background(), "", time.Now())
if err != nil {
t.Fatal(err)
}
var out []protocol.AppNotification
for _, n := range active {
if n.Kind == kind {
out = append(out, n)
}
}
return out
}
func stepFrameFor(status, turn string) protocol.ChatEventFrame {
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: status, TurnID: turn}}
}
func textFrame(text string) protocol.ChatEventFrame {
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventText, MessageID: "msg_a", Part: &protocol.ChatPart{ID: "msg_a_p"}, Delta: text}
}
// A turn started from the app keeps being observed after the app closes, so
// its finished reply is recorded (app stream) and published (ntfy) once, and
// the observer is torn down when the turn ends.
func TestChatReplyNotifiesAfterLastClientLeaves(t *testing.T) {
fastUnattended(t)
captured := newNtfyCapture(t)
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
waitFor(t, "prompt", func() bool { return broker.promptCount() == 1 })
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
broker.push(textFrame("The build is green."))
conn.CloseNow()
time.Sleep(100 * time.Millisecond) // let the server notice the disconnect
if active, _ := broker.streams(); active != 1 {
t.Fatalf("in-flight turn not observed after the client left: %d streams", active)
}
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
waitFor(t, "ntfy publish", func() bool { return len(captured()) == 1 })
time.Sleep(50 * time.Millisecond)
notes := chatNotes(t, store, protocol.NotificationChatReply)
if got := captured(); len(notes) != 1 || len(got) != 1 || got[0].Title != "Release prep replied" || got[0].Message != "The build is green." {
t.Fatalf("notes=%+v ntfy=%+v", notes, got)
}
if notes[0].Message != "The build is green." || notes[0].Click != "pipelines://chat/ses_1" {
t.Fatalf("note = %+v", notes[0])
}
}
// The chat_reply_finished setting still decides: off (the default) records
// nothing, even though the unattended turn is observed.
func TestChatUnattendedReplyRespectsSetting(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, false)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
time.Sleep(100 * time.Millisecond)
if notes := chatNotes(t, store, protocol.NotificationChatReply); len(notes) != 0 {
t.Fatalf("reply recorded with chat_reply_finished off: %+v", notes)
}
}
// A question the agent asks while no client is attached notifies.
func TestChatQuestionNotifiesWithNoClient(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, false)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "ship?", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Which branch should ship?"}
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
if notes := chatNotes(t, store, protocol.NotificationChatQuestion); notes[0].Message != "Which branch should ship?" {
t.Fatalf("note = %+v", notes[0])
}
// The turn ends with the question withdrawn: nothing is left to observe.
form.Cancelled = true
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A client reattaching mid-turn joins the observing room: the reply is still
// announced exactly once and the client sees the turn end.
func TestChatReattachMidTurnDoesNotDuplicate(t *testing.T) {
fastUnattended(t)
captured := newNtfyCapture(t)
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
broker.push(textFrame("Done."))
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Continue?"}
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
again := dialChatConn(t, server.ctx, server.srv)
writeChat(t, server.ctx, again, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, server.ctx, again, protocol.ChatSubscribe)
if _, opened := broker.streams(); opened != 1 {
t.Fatalf("reattach opened a second stream: %d", opened)
}
form.Resolved = true
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
for {
msg := readUntil(t, server.ctx, again, protocol.ChatEvent)
if e := msg.ChatEvent; e.Kind == protocol.ChatEventStep && e.Step != nil && e.Step.Status == protocol.ChatStepEnded {
break
}
}
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
time.Sleep(100 * time.Millisecond)
if got := captured(); len(got) != 2 {
t.Fatalf("ntfy deliveries = %+v, want one question and one reply", got)
}
if n := len(chatNotes(t, store, protocol.NotificationChatQuestion)) + len(chatNotes(t, store, protocol.NotificationChatReply)); n > 2 {
t.Fatalf("duplicate notifications: %d", n)
}
// The attended room is idle now; the client leaving releases it.
again.CloseNow()
waitFor(t, "idle room released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// An idle session is not observed once its last client leaves, and a server
// start does not observe sessions whose harness reports no running turn.
func TestChatIdleSessionWithNoClientIsNotObserved(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
waitFor(t, "start-up probe", func() bool {
broker.smu.Lock()
defer broker.smu.Unlock()
return broker.probes == 1
})
if _, opened := broker.streams(); opened != 0 {
t.Fatalf("idle session observed at start: %d streams", opened)
}
conn := dialChatConn(t, server.ctx, server.srv)
writeChat(t, server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, server.ctx, conn, protocol.ChatSubscribe)
waitFor(t, "stream", func() bool { active, _ := broker.streams(); return active == 1 })
conn.CloseNow()
waitFor(t, "idle stream released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A turn a previous server process left running is observed again at start,
// with no client, so its reply still notifies; then the observer goes away.
func TestChatRunningTurnObservedAfterRestart(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
broker.probe["ses_1"] = true
startQueueServer(t, broker, store)
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
// The harness re-announces the running turn on attach, then finishes it.
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_9"))
broker.push(textFrame("Finished after the restart."))
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_9"))
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A recovered session whose stream never reports the turn is released after
// the settle window rather than observed forever.
func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
broker.probe["ses_1"] = true
startQueueServer(t, broker, store)
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
waitFor(t, "released after settle", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A turn that never reports its end does not hold an observer forever.
func TestChatStuckUnattendedTurnIsReleased(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
limit := chatUnattendedTurnLimit
chatUnattendedTurnLimit = 300 * time.Millisecond
t.Cleanup(func() { chatUnattendedTurnLimit = limit })
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
if active, _ := broker.streams(); active != 1 {
t.Fatalf("in-flight turn not observed: %d", active)
}
waitFor(t, "stuck observer released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// At most chatUnattendedLimit chats are kept observing with no client.
func TestChatUnattendedRoomsAreBounded(t *testing.T) {
fastUnattended(t)
h := newChatHub()
defer func() {
h.mu.Lock()
for _, room := range h.rooms {
room.cancel()
}
h.mu.Unlock()
}()
kept := 0
for i := 0; i < chatUnattendedLimit+3; i++ {
id := "ses_" + string(rune('a'+i))
h.mu.Lock()
room := h.roomLocked(id)
room.running = true
sub := &chatSub{hub: h, room: room, clientID: "c", ch: make(chan protocol.Message, 1), done: make(chan struct{})}
room.subs["c"] = sub
h.mu.Unlock()
h.detach(id, sub)
h.mu.Lock()
if h.rooms[id] == room {
kept++
}
h.mu.Unlock()
}
if kept != chatUnattendedLimit {
t.Fatalf("kept %d unattended rooms, want %d", kept, chatUnattendedLimit)
}
}
+5
View File
@@ -115,6 +115,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
settingsStore = stores[0]
}
hub.settings = settingsStore
if options.Chat != nil && registry != nil {
// Keep observing chats whose turn a previous process left running, so
// their reply or question still notifies with no client attached.
go hub.observeRunningTurns(ctx, options.Chat)
}
if len(stores) > 0 {
// Prompts queued behind a running turn are durable and owned by their
// session; resume delivering any a previous process left queued.