Keep observing chats with an in-flight turn after the last client leaves
Chat attention notifications (questions, approvals, finished replies) come from a session's event stream, which the hub only read while a room existed; detach destroyed the room when the last client left, so a turn that finished or asked with the app closed notified nobody. A room whose last client leaves now stays observing, unattended, while its turn is in flight or it holds an unanswered question/approval, and is torn down when the turn ends (or after 6h for a stuck turn); at most 16 such rooms. At server start, recently used open sessions whose harness reports a running turn (OpenCode /api/session/active, Codex thread/turns/list) are observed again so their reply still notifies. Settings, dedupe and app_notifications recording are unchanged.
This commit is contained in:
+17
-3
@@ -21,9 +21,23 @@ The same settings decide what is sent on both paths.
|
||||
Each event notifies once. Opening or dismissing it on one device clears it on
|
||||
the others.
|
||||
|
||||
Chat events are seen while some client has the chat open or the server still
|
||||
holds queued messages for it. A turn that finishes with no client attached and
|
||||
nothing queued does not produce a "replied" notification.
|
||||
Chat events do not need the chat to be open. When the last app or browser
|
||||
leaves a chat whose turn is still running, or which has an unanswered question
|
||||
or approval, the server keeps watching it until the turn ends and nothing is
|
||||
left to answer, so "Chat needs you" and "Chat replied" still arrive with the
|
||||
app closed. A client that reopens the chat meanwhile joins the same watch, so
|
||||
nothing is announced twice. Idle chats are not watched.
|
||||
|
||||
Limits:
|
||||
|
||||
- At most 16 chats are watched with no client attached; a further one is not
|
||||
(queued messages are always delivered regardless).
|
||||
- A turn that never reports its end stops being watched after 6 hours.
|
||||
- After a server restart, the 32 most recently used open chats (used in the last
|
||||
24 hours) are asked whether a turn is still running, and those that are are
|
||||
watched again. This works for OpenCode and Codex, which keep running across a
|
||||
Pipelines restart. A Claude Code turn runs inside the server process and ends
|
||||
with it, so there is nothing to resume.
|
||||
|
||||
## Settings
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package codex
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ProbeTurn reports the latest turn's in-progress state from the App Server.
|
||||
func TestProbeTurnReadsLatestTurn(t *testing.T) {
|
||||
var completed atomic.Bool
|
||||
c := modelTestChat(t, func(method string, _ json.RawMessage) any {
|
||||
if method != "thread/turns/list" {
|
||||
t.Errorf("unexpected method: %s", method)
|
||||
}
|
||||
status := "inProgress"
|
||||
if completed.Load() {
|
||||
status = "completed"
|
||||
}
|
||||
return json.RawMessage(`{"data":[{"id":"turn","status":"` + status + `","items":[]}],"nextCursor":null}`)
|
||||
})
|
||||
if running, err := c.ProbeTurn(context.Background(), fixtureThread); err != nil || !running {
|
||||
t.Fatalf("in-progress turn: %v %v", running, err)
|
||||
}
|
||||
completed.Store(true)
|
||||
if running, err := c.ProbeTurn(context.Background(), fixtureThread); err != nil || running {
|
||||
t.Fatalf("completed turn: %v %v", running, err)
|
||||
}
|
||||
}
|
||||
@@ -121,3 +121,14 @@ func (c *Chat) streamWithReadback(ctx context.Context, id string, ready func(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ProbeTurn reports whether the thread's latest turn is still in progress in
|
||||
// the App Server, which outlives a Pipelines restart. The server uses it once
|
||||
// at start to resume observing a turn a previous process left running.
|
||||
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
|
||||
last, err := c.latestTurn(ctx, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return activeTurn(last.turn), nil
|
||||
}
|
||||
|
||||
@@ -1863,3 +1863,25 @@ func decodeChatSession(data []byte) (protocol.ChatSession, error) {
|
||||
}
|
||||
return chatSessionFields(record), nil
|
||||
}
|
||||
|
||||
// ProbeTurn reports whether OpenCode is running a turn for the session, read
|
||||
// from the same /api/session/active list Stream polls. The server uses it once
|
||||
// at start to resume observing turns a previous process left running. A failed
|
||||
// read is an error, never "idle".
|
||||
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
|
||||
if !sessionID.MatchString(id) {
|
||||
return false, fmt.Errorf("invalid OpenCode session id")
|
||||
}
|
||||
data, err := c.call(ctx, "get", "/api/session/active", nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var response struct {
|
||||
Data *map[string]json.RawMessage `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
|
||||
return false, fmt.Errorf("invalid OpenCode active-session readback")
|
||||
}
|
||||
_, active := (*response.Data)[id]
|
||||
return active, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ProbeTurn reads the active-session list; a failed or malformed read is an
|
||||
// error, never "idle".
|
||||
func TestChatProbeTurn(t *testing.T) {
|
||||
var malformed atomic.Bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/session/active" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if malformed.Load() {
|
||||
fmt.Fprint(w, `{"data":null}`)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, `{"data":{"ses_1":{"type":"running"}}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
chat := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
ctx := context.Background()
|
||||
if running, err := chat.ProbeTurn(ctx, "ses_1"); err != nil || !running {
|
||||
t.Fatalf("active session: %v %v", running, err)
|
||||
}
|
||||
if running, err := chat.ProbeTurn(ctx, "ses_2"); err != nil || running {
|
||||
t.Fatalf("idle session: %v %v", running, err)
|
||||
}
|
||||
malformed.Store(true)
|
||||
if _, err := chat.ProbeTurn(ctx, "ses_1"); err == nil {
|
||||
t.Fatal("malformed readback treated as idle")
|
||||
}
|
||||
if _, err := chat.ProbeTurn(ctx, "bad id"); err == nil {
|
||||
t.Fatal("invalid id accepted")
|
||||
}
|
||||
}
|
||||
@@ -100,6 +100,14 @@ func TestCompactionTransportLossAndInterruptClearDoNotReleaseTurn(t *testing.T)
|
||||
room.cancel()
|
||||
h.drainStream(room, result)
|
||||
}
|
||||
if mode == "detach" {
|
||||
// The last client leaving a running turn keeps the room observing
|
||||
// unattended: the turn, and its live compaction, are still current.
|
||||
if h.rooms[room.sessionID] != room || !room.watching || !room.running {
|
||||
t.Fatal("detach released the room of a live turn")
|
||||
}
|
||||
return
|
||||
}
|
||||
if h.turns[room.sessionID].compaction != nil || !room.running {
|
||||
t.Fatal("cleanup stuck label or released live turn")
|
||||
}
|
||||
|
||||
@@ -58,6 +58,9 @@ type chatHub struct {
|
||||
// operation ids, so a client retry is acknowledged instead of re-sent.
|
||||
recentOps map[string]struct{}
|
||||
recentOpOrder []string
|
||||
// unattended holds the unattended-observation timings (chat_unattended.go),
|
||||
// fixed when the hub is built.
|
||||
unattended chatUnattendedTimings
|
||||
}
|
||||
|
||||
// chatPart is the retained form of one part. Text and reasoning deltas append to
|
||||
@@ -121,7 +124,7 @@ type chatResolution struct {
|
||||
}
|
||||
|
||||
func newChatHub() *chatHub {
|
||||
return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now}
|
||||
return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now, unattended: currentChatUnattendedTimings()}
|
||||
}
|
||||
|
||||
// nextUserMessageID mints a stable id for one echoed user turn.
|
||||
|
||||
@@ -701,3 +701,19 @@ func (m *chatManager) TurnRunning(sessionID string) bool {
|
||||
reporter, ok := adapter.(chatTurnReporter)
|
||||
return ok && reporter.TurnRunning(sessionID)
|
||||
}
|
||||
|
||||
// ProbeTurn asks the session's own harness whether a turn is in flight. It is
|
||||
// the server-start probe (chat_unattended.go): a harness that can be asked
|
||||
// (OpenCode, Codex) is asked; otherwise its in-memory state is used, which for
|
||||
// Claude Code (a child of this server) is empty after a restart.
|
||||
func (m *chatManager) ProbeTurn(ctx context.Context, sessionID string) (bool, error) {
|
||||
adapter, err := m.brokerFor(ctx, sessionID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if prober, ok := adapter.(chatTurnProber); ok {
|
||||
return prober.ProbeTurn(ctx, sessionID)
|
||||
}
|
||||
reporter, ok := adapter.(chatTurnReporter)
|
||||
return ok && reporter.TurnRunning(sessionID), nil
|
||||
}
|
||||
|
||||
@@ -167,6 +167,14 @@ type chatRoom struct {
|
||||
// notBefore holds back a recovered queue until the restarted stream had a
|
||||
// chance to report a turn the harness is still running.
|
||||
notBefore time.Time
|
||||
// watching is set while the unattended watcher (chat_unattended.go) owns
|
||||
// this room's lifetime: the room has no client but is kept observing so a
|
||||
// finished turn or a question still notifies. unattendedSince bounds how
|
||||
// long a stuck turn may hold it; holdUntil keeps a room recovered at start
|
||||
// alive until its stream has had a chance to report the running turn.
|
||||
watching bool
|
||||
unattendedSince time.Time
|
||||
holdUntil time.Time
|
||||
}
|
||||
|
||||
func newChatRoom(base context.Context, sessionID string) *chatRoom {
|
||||
@@ -309,11 +317,13 @@ func (h *chatHub) attach(sessionID, clientID, label string, broker chatBroker, w
|
||||
}
|
||||
|
||||
// detach drops one subscription. The room (and its broker stream and queue) is
|
||||
// released once its last client leaves and it has no prompt left to deliver,
|
||||
// so a session with no watchers holds no goroutine. A room holding queued
|
||||
// released once its last client leaves and it has nothing left to do, so an
|
||||
// idle session with no watchers holds no goroutine. A room holding queued
|
||||
// prompts stays (with its stream) until its runner has delivered them: the
|
||||
// queue belongs to the session, not to the clients watching it. A running turn
|
||||
// with nothing queued behind it needs no room; the harness finishes it alone.
|
||||
// queue belongs to the session, not to the clients watching it. A room whose
|
||||
// turn is still in flight, or which holds an unanswered question or approval,
|
||||
// also stays (bounded, see chat_unattended.go) so the operator is still
|
||||
// notified when the turn finishes or the agent asks while no app is open.
|
||||
func (h *chatHub) detach(sessionID string, sub *chatSub) {
|
||||
h.mu.Lock()
|
||||
room := h.rooms[sessionID]
|
||||
@@ -322,8 +332,8 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) {
|
||||
return
|
||||
}
|
||||
delete(room.subs, sub.clientID)
|
||||
if len(room.subs) == 0 && len(room.queue) == 0 {
|
||||
h.releaseRoomLocked(room)
|
||||
if len(room.subs) == 0 {
|
||||
h.leaveUnattendedLocked(room)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
@@ -461,6 +471,11 @@ func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
|
||||
}
|
||||
if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded && !frame.Step.Intermediate {
|
||||
h.finishTurnLocked(room)
|
||||
// An unwatched room kept only for this turn is done once it ends (its
|
||||
// reply attention was noted above).
|
||||
if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) {
|
||||
h.releaseRoomLocked(room)
|
||||
}
|
||||
}
|
||||
// Shell replay and live lifecycle delivery must enqueue in the same order
|
||||
// as folding. Otherwise an exit can overtake an older running replay.
|
||||
@@ -561,10 +576,12 @@ func (h *chatHub) finishTurnLocked(room *chatRoom) {
|
||||
|
||||
// turnInFlight reports whether sessionID has a turn in flight, so chat_close
|
||||
// knows whether to stop one through the broker interrupt before it records the
|
||||
// closed state. A room exists only while a client is watching the session, and
|
||||
// running is set by the turn runner for the whole turn (a prompt queued behind a
|
||||
// closed state. A room exists only while a client is watching the session or
|
||||
// it still has work, and running is set by the turn runner for the whole turn (a prompt queued behind a
|
||||
// live turn leaves it set), so an idle or unwatched session reports false and no
|
||||
// interrupt is sent. It is one read under the hub mutex and mutates nothing.
|
||||
// interrupt is sent. (A room also outlives its last client while its turn is in
|
||||
// flight, see chat_unattended.go.) It is one read under the hub mutex and
|
||||
// mutates nothing.
|
||||
func (h *chatHub) turnInFlight(sessionID string) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
@@ -709,10 +726,15 @@ func (h *chatHub) runRoom(room *chatRoom) {
|
||||
}
|
||||
h.runTurn(room, p, done)
|
||||
// The last client may have left while this turn ran; an unwatched room
|
||||
// stays only while it still has prompts to deliver.
|
||||
// stays only while it still has prompts to deliver or something the
|
||||
// operator must be told about.
|
||||
h.mu.Lock()
|
||||
if len(room.subs) == 0 && !room.running && len(room.queue) == 0 {
|
||||
h.releaseRoomLocked(room)
|
||||
if len(room.subs) == 0 && len(room.queue) == 0 {
|
||||
if h.keepUnattendedLocked(room, false, true) {
|
||||
h.watchUnattendedLocked(room)
|
||||
} else {
|
||||
h.releaseRoomLocked(room)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -770,12 +770,14 @@ func TestChatMessagesRejectsInvalidSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The stream of a session whose turn is still in flight outlives the client
|
||||
// (so its reply still notifies), but no goroutine outlives the server.
|
||||
func TestChatDisconnectCancelsStream(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
broker.started = make(chan struct{})
|
||||
broker.done = make(chan struct{})
|
||||
ctx, _, conn := dialChat(t, broker)
|
||||
ctx, cancel, conn := dialChat(t, broker)
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: "r1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -789,12 +791,17 @@ func TestChatDisconnectCancelsStream(t *testing.T) {
|
||||
t.Fatal("stream never started")
|
||||
}
|
||||
conn.CloseNow()
|
||||
// The server's read loop returns on disconnect, canceling the connection
|
||||
// context the subscription observes; no goroutine may keep running.
|
||||
// The turn is still in flight, so the session stays observed unattended.
|
||||
select {
|
||||
case <-broker.done:
|
||||
t.Fatal("chat stream of an in-flight turn was canceled on disconnect")
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-broker.done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("chat stream was not canceled on disconnect")
|
||||
t.Fatal("chat stream was not canceled on shutdown")
|
||||
}
|
||||
if prompts, _, _ := broker.counts(); prompts != 1 {
|
||||
t.Fatalf("prompt forwarded %d times, want 1", prompts)
|
||||
@@ -2205,9 +2212,9 @@ func TestChatPermissionDedupesRepeatedPolls(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatDisconnectCancelsPermissionPoller pins that the permission poller is
|
||||
// canceled on disconnect (reusing the stream's context), so it cannot leak a
|
||||
// goroutine after the client goes away.
|
||||
// TestChatDisconnectCancelsPermissionPoller pins that the permission poller
|
||||
// shares the room's context: it survives a disconnect while the turn is in
|
||||
// flight and cannot leak a goroutine past the server.
|
||||
func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
@@ -2215,7 +2222,7 @@ func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
|
||||
broker.done = make(chan struct{})
|
||||
broker.permBlock = true
|
||||
broker.permDone = make(chan struct{})
|
||||
ctx, _, conn := dialChat(t, broker)
|
||||
ctx, cancel, conn := dialChat(t, broker)
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: "r1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -2234,10 +2241,18 @@ func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
|
||||
default:
|
||||
}
|
||||
conn.CloseNow()
|
||||
// An in-flight turn keeps its poller (an approval it raises still
|
||||
// notifies); shutting the server down cancels it.
|
||||
select {
|
||||
case <-broker.permDone:
|
||||
t.Fatal("permission poller of an in-flight turn was canceled on disconnect")
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-broker.permDone:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("permission poller was not canceled on disconnect")
|
||||
t.Fatal("permission poller was not canceled on shutdown")
|
||||
}
|
||||
select {
|
||||
case <-broker.done:
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Chat attention notifications (chat_question, chat_approval, chat_reply) are
|
||||
// derived from a session's live event stream, and the hub only reads that
|
||||
// stream while the session has a room. A room normally lives while a client
|
||||
// watches the chat, but the operator relies on these notifications precisely
|
||||
// when no app is open. So a room whose last client leaves stays observing,
|
||||
// unattended, while its turn is in flight or it holds an unanswered question
|
||||
// or approval, and is torn down once that work is done. The same unattended
|
||||
// observation is started at server start for sessions whose harness reports a
|
||||
// running turn, so a reply that finishes after a restart still notifies.
|
||||
//
|
||||
// Resources are bounded: at most chatUnattendedLimit rooms are kept without a
|
||||
// client (rooms holding queued prompts are not counted: they were always kept
|
||||
// until delivered), a stuck turn is released after chatUnattendedTurnLimit,
|
||||
// and an idle session is never observed.
|
||||
|
||||
// chatUnattendedLimit bounds the rooms kept observing with no client attached
|
||||
// and no queued prompt.
|
||||
const chatUnattendedLimit = 16
|
||||
|
||||
// chatRecoverSessionLimit bounds how many recently used open sessions are
|
||||
// probed for a running turn at server start.
|
||||
const chatRecoverSessionLimit = 32
|
||||
|
||||
var (
|
||||
// chatUnattendedCheck is how often an unattended room re-checks whether it
|
||||
// still has a reason to observe its session.
|
||||
chatUnattendedCheck = 5 * time.Second
|
||||
// chatUnattendedTurnLimit releases an unattended room whose turn never
|
||||
// reports an end (a stuck harness or a lost end event).
|
||||
chatUnattendedTurnLimit = 6 * time.Hour
|
||||
// chatUnattendedRecoverSettle keeps a room recovered at start alive until
|
||||
// its freshly attached stream has had time to announce the running turn.
|
||||
chatUnattendedRecoverSettle = 15 * time.Second
|
||||
// chatRecoverWindow limits start-up probing to sessions used recently.
|
||||
chatRecoverWindow = 24 * time.Hour
|
||||
)
|
||||
|
||||
// chatUnattendedTimings is the hub's copy of the timings above, taken when the
|
||||
// hub is built so its goroutines never read the package variables.
|
||||
type chatUnattendedTimings struct {
|
||||
check, turnLimit, settle, window time.Duration
|
||||
}
|
||||
|
||||
func currentChatUnattendedTimings() chatUnattendedTimings {
|
||||
return chatUnattendedTimings{check: chatUnattendedCheck, turnLimit: chatUnattendedTurnLimit, settle: chatUnattendedRecoverSettle, window: chatRecoverWindow}
|
||||
}
|
||||
|
||||
// chatTurnProber is implemented by a broker that can ask the harness itself,
|
||||
// with I/O, whether a session's turn is in flight. It is used once at server
|
||||
// start, when the in-memory turn state is empty.
|
||||
type chatTurnProber interface {
|
||||
ProbeTurn(ctx context.Context, sessionID string) (bool, error)
|
||||
}
|
||||
|
||||
// keepUnattendedLocked reports whether a room with no client still has work
|
||||
// the operator must hear about: a turn in flight (the runner's, or one its
|
||||
// stream announced), an unresolved approval or question, or a harness that
|
||||
// reports its own turn running (busy, read by the caller off the hub mutex).
|
||||
// hold honours the start-up settle window. Queued prompts are the caller's
|
||||
// concern. Callers hold h.mu.
|
||||
func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool {
|
||||
if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 {
|
||||
return true
|
||||
}
|
||||
return hold && h.clock().Before(room.holdUntil)
|
||||
}
|
||||
|
||||
// unattendedCountLocked counts the rooms kept observing with no client and no
|
||||
// queued prompt, other than except. Callers hold h.mu.
|
||||
func (h *chatHub) unattendedCountLocked(except *chatRoom) int {
|
||||
n := 0
|
||||
for _, room := range h.rooms {
|
||||
if room != except && room.watching && len(room.subs) == 0 && len(room.queue) == 0 {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// leaveUnattendedLocked decides the fate of a room whose last client just
|
||||
// left: a room holding queued prompts stays for its runner (as before); a room
|
||||
// with an in-flight turn or pending attention keeps observing unattended,
|
||||
// within the bound; anything else is released. Callers hold h.mu.
|
||||
func (h *chatHub) leaveUnattendedLocked(room *chatRoom) {
|
||||
if len(room.queue) > 0 {
|
||||
h.watchUnattendedLocked(room)
|
||||
return
|
||||
}
|
||||
if !h.keepUnattendedLocked(room, false, true) {
|
||||
h.releaseRoomLocked(room)
|
||||
return
|
||||
}
|
||||
if !room.watching && h.unattendedCountLocked(room) >= chatUnattendedLimit {
|
||||
log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit)
|
||||
h.releaseRoomLocked(room)
|
||||
return
|
||||
}
|
||||
h.watchUnattendedLocked(room)
|
||||
}
|
||||
|
||||
// watchUnattendedLocked starts the room's unattended watcher once. Callers
|
||||
// hold h.mu.
|
||||
func (h *chatHub) watchUnattendedLocked(room *chatRoom) {
|
||||
if room.watching || room.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
room.watching = true
|
||||
room.unattendedSince = h.clock()
|
||||
go h.watchUnattended(room)
|
||||
}
|
||||
|
||||
func (h *chatHub) watchUnattended(room *chatRoom) {
|
||||
check := h.unattended.check
|
||||
if check <= 0 {
|
||||
check = 5 * time.Second
|
||||
}
|
||||
ticker := time.NewTicker(check)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-room.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
if h.reapUnattended(room) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reapUnattended re-checks one unattended room and reports whether its
|
||||
// watcher is finished: a client reattached (the room is attended again), or
|
||||
// the room had nothing left to observe, or its turn exceeded the limit, and
|
||||
// was released.
|
||||
func (h *chatHub) reapUnattended(room *chatRoom) bool {
|
||||
h.mu.Lock()
|
||||
if done := h.attendedLocked(room); done {
|
||||
h.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
broker := room.broker
|
||||
h.mu.Unlock()
|
||||
// TurnRunning performs no I/O but may take the broker's own lock, so it is
|
||||
// never asked under the hub mutex.
|
||||
busy := brokerTurnRunning(broker, room.sessionID)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if done := h.attendedLocked(room); done {
|
||||
return true
|
||||
}
|
||||
if len(room.queue) > 0 {
|
||||
// The runner owns a room with queued prompts until they are delivered;
|
||||
// the stuck-turn limit counts from the last of them.
|
||||
room.unattendedSince = h.clock()
|
||||
return false
|
||||
}
|
||||
expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit
|
||||
if !expired && h.keepUnattendedLocked(room, busy, true) {
|
||||
return false
|
||||
}
|
||||
if expired {
|
||||
log.Printf("chat session %s: released the unattended observer after %s without the turn ending", room.sessionID, h.unattended.turnLimit)
|
||||
}
|
||||
room.watching = false
|
||||
h.releaseRoomLocked(room)
|
||||
return true
|
||||
}
|
||||
|
||||
// attendedLocked reports whether the watcher should stop because the room is
|
||||
// gone or a client is watching it again. Callers hold h.mu.
|
||||
func (h *chatHub) attendedLocked(room *chatRoom) bool {
|
||||
if h.rooms[room.sessionID] != room || room.ctx.Err() != nil {
|
||||
room.watching = false
|
||||
return true
|
||||
}
|
||||
if len(room.subs) > 0 {
|
||||
room.watching = false
|
||||
room.unattendedSince = time.Time{}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// observeRunningTurns is the start-up half: each recently used open operator
|
||||
// session whose harness reports a turn in flight gets an unattended room
|
||||
// observing its stream, so the reply (or a question) still notifies. It is
|
||||
// best effort per harness: OpenCode and Codex keep running across a Pipelines
|
||||
// restart and report it; Claude Code runs as a child of the server and so has
|
||||
// no turn left after a restart.
|
||||
func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
|
||||
prober, ok := broker.(chatTurnProber)
|
||||
if !ok || h.registry == nil {
|
||||
return
|
||||
}
|
||||
sessions, err := h.registry.OperatorSessions(ctx)
|
||||
if err != nil {
|
||||
log.Printf("chat turn recovery: %v", err)
|
||||
return
|
||||
}
|
||||
cutoff := h.clock().Add(-h.unattended.window).UnixMilli()
|
||||
probed := 0
|
||||
for _, session := range sessions {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if session.ClosedAt != 0 || session.UpdatedAt < cutoff || !validChatSessionID(session.ID) {
|
||||
continue
|
||||
}
|
||||
if probed >= chatRecoverSessionLimit {
|
||||
break
|
||||
}
|
||||
probed++
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
running, err := prober.ProbeTurn(probeCtx, session.ID)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("chat turn recovery probe session=%s failed (%T)", session.ID, err)
|
||||
continue
|
||||
}
|
||||
if running {
|
||||
h.observeUnattended(session.ID, broker)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// observeUnattended opens (or reuses) the session's room with no client and
|
||||
// observes its stream until the turn the harness reported has ended.
|
||||
func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
|
||||
h.mu.Lock()
|
||||
if h.rooms[sessionID] == nil && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
|
||||
h.mu.Unlock()
|
||||
log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit)
|
||||
return
|
||||
}
|
||||
room := h.roomLocked(sessionID)
|
||||
if room.broker == nil {
|
||||
room.broker = broker
|
||||
}
|
||||
if len(room.subs) == 0 {
|
||||
room.holdUntil = h.clock().Add(h.unattended.settle)
|
||||
h.watchUnattendedLocked(room)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if err := h.ensureObserving(room, broker); err != nil {
|
||||
if room.ctx.Err() == nil {
|
||||
log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
h.startRoomPoller(room, broker)
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// observedChat is a channelChat whose stream can be opened more than once and
|
||||
// reports how many streams are live, so a test can prove when the hub observes
|
||||
// a session with no client attached and when it stops. probe answers the
|
||||
// server-start ProbeTurn.
|
||||
type observedChat struct {
|
||||
*channelChat
|
||||
smu sync.Mutex
|
||||
active int
|
||||
opened int
|
||||
probe map[string]bool
|
||||
probes int
|
||||
}
|
||||
|
||||
func newObservedChat() *observedChat {
|
||||
return &observedChat{channelChat: newChannelChat(), probe: map[string]bool{}}
|
||||
}
|
||||
|
||||
func (c *observedChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
c.smu.Lock()
|
||||
c.active++
|
||||
c.opened++
|
||||
c.smu.Unlock()
|
||||
defer func() {
|
||||
c.smu.Lock()
|
||||
c.active--
|
||||
c.smu.Unlock()
|
||||
}()
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case frame := <-c.frames:
|
||||
if err := emit(frame); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *observedChat) ProbeTurn(_ context.Context, sessionID string) (bool, error) {
|
||||
c.smu.Lock()
|
||||
defer c.smu.Unlock()
|
||||
c.probes++
|
||||
return c.probe[sessionID], nil
|
||||
}
|
||||
|
||||
func (c *observedChat) streams() (active, opened int) {
|
||||
c.smu.Lock()
|
||||
defer c.smu.Unlock()
|
||||
return c.active, c.opened
|
||||
}
|
||||
|
||||
func fastUnattended(t *testing.T) {
|
||||
t.Helper()
|
||||
check, settle := chatUnattendedCheck, chatUnattendedRecoverSettle
|
||||
chatUnattendedCheck = 10 * time.Millisecond
|
||||
chatUnattendedRecoverSettle = 200 * time.Millisecond
|
||||
t.Cleanup(func() { chatUnattendedCheck, chatUnattendedRecoverSettle = check, settle })
|
||||
}
|
||||
|
||||
func unattendedStore(t *testing.T, replies bool) *pipeline.Store {
|
||||
t.Helper()
|
||||
store := openQueueStore(t, filepath.Join(t.TempDir(), "chat.db"))
|
||||
if err := store.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if replies {
|
||||
if _, err := store.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// chatNotes returns the recorded app notifications of one kind.
|
||||
func chatNotes(t *testing.T, store *pipeline.Store, kind string) []protocol.AppNotification {
|
||||
t.Helper()
|
||||
active, _, err := store.ActiveAppNotifications(context.Background(), "", time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []protocol.AppNotification
|
||||
for _, n := range active {
|
||||
if n.Kind == kind {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stepFrameFor(status, turn string) protocol.ChatEventFrame {
|
||||
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: status, TurnID: turn}}
|
||||
}
|
||||
|
||||
func textFrame(text string) protocol.ChatEventFrame {
|
||||
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventText, MessageID: "msg_a", Part: &protocol.ChatPart{ID: "msg_a_p"}, Delta: text}
|
||||
}
|
||||
|
||||
// A turn started from the app keeps being observed after the app closes, so
|
||||
// its finished reply is recorded (app stream) and published (ntfy) once, and
|
||||
// the observer is torn down when the turn ends.
|
||||
func TestChatReplyNotifiesAfterLastClientLeaves(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
captured := newNtfyCapture(t)
|
||||
store := unattendedStore(t, true)
|
||||
broker := newObservedChat()
|
||||
server := startQueueServer(t, broker, store)
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
||||
waitFor(t, "prompt", func() bool { return broker.promptCount() == 1 })
|
||||
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
||||
broker.push(textFrame("The build is green."))
|
||||
conn.CloseNow()
|
||||
time.Sleep(100 * time.Millisecond) // let the server notice the disconnect
|
||||
if active, _ := broker.streams(); active != 1 {
|
||||
t.Fatalf("in-flight turn not observed after the client left: %d streams", active)
|
||||
}
|
||||
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
||||
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
|
||||
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
waitFor(t, "ntfy publish", func() bool { return len(captured()) == 1 })
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
notes := chatNotes(t, store, protocol.NotificationChatReply)
|
||||
if got := captured(); len(notes) != 1 || len(got) != 1 || got[0].Title != "Release prep replied" || got[0].Message != "The build is green." {
|
||||
t.Fatalf("notes=%+v ntfy=%+v", notes, got)
|
||||
}
|
||||
if notes[0].Message != "The build is green." || notes[0].Click != "pipelines://chat/ses_1" {
|
||||
t.Fatalf("note = %+v", notes[0])
|
||||
}
|
||||
}
|
||||
|
||||
// The chat_reply_finished setting still decides: off (the default) records
|
||||
// nothing, even though the unattended turn is observed.
|
||||
func TestChatUnattendedReplyRespectsSetting(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
store := unattendedStore(t, false)
|
||||
broker := newObservedChat()
|
||||
server := startQueueServer(t, broker, store)
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
||||
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
||||
conn.CloseNow()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
||||
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if notes := chatNotes(t, store, protocol.NotificationChatReply); len(notes) != 0 {
|
||||
t.Fatalf("reply recorded with chat_reply_finished off: %+v", notes)
|
||||
}
|
||||
}
|
||||
|
||||
// A question the agent asks while no client is attached notifies.
|
||||
func TestChatQuestionNotifiesWithNoClient(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
store := unattendedStore(t, false)
|
||||
broker := newObservedChat()
|
||||
server := startQueueServer(t, broker, store)
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
sendPrompt(t, server.ctx, conn, "p1", "ship?", "op-1")
|
||||
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
||||
conn.CloseNow()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Which branch should ship?"}
|
||||
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
|
||||
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
|
||||
if notes := chatNotes(t, store, protocol.NotificationChatQuestion); notes[0].Message != "Which branch should ship?" {
|
||||
t.Fatalf("note = %+v", notes[0])
|
||||
}
|
||||
// The turn ends with the question withdrawn: nothing is left to observe.
|
||||
form.Cancelled = true
|
||||
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
|
||||
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
||||
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
}
|
||||
|
||||
// A client reattaching mid-turn joins the observing room: the reply is still
|
||||
// announced exactly once and the client sees the turn end.
|
||||
func TestChatReattachMidTurnDoesNotDuplicate(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
captured := newNtfyCapture(t)
|
||||
store := unattendedStore(t, true)
|
||||
broker := newObservedChat()
|
||||
server := startQueueServer(t, broker, store)
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
||||
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
||||
broker.push(textFrame("Done."))
|
||||
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Continue?"}
|
||||
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
|
||||
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
|
||||
conn.CloseNow()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
again := dialChatConn(t, server.ctx, server.srv)
|
||||
writeChat(t, server.ctx, again, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
|
||||
readUntil(t, server.ctx, again, protocol.ChatSubscribe)
|
||||
if _, opened := broker.streams(); opened != 1 {
|
||||
t.Fatalf("reattach opened a second stream: %d", opened)
|
||||
}
|
||||
form.Resolved = true
|
||||
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
|
||||
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
||||
for {
|
||||
msg := readUntil(t, server.ctx, again, protocol.ChatEvent)
|
||||
if e := msg.ChatEvent; e.Kind == protocol.ChatEventStep && e.Step != nil && e.Step.Status == protocol.ChatStepEnded {
|
||||
break
|
||||
}
|
||||
}
|
||||
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if got := captured(); len(got) != 2 {
|
||||
t.Fatalf("ntfy deliveries = %+v, want one question and one reply", got)
|
||||
}
|
||||
if n := len(chatNotes(t, store, protocol.NotificationChatQuestion)) + len(chatNotes(t, store, protocol.NotificationChatReply)); n > 2 {
|
||||
t.Fatalf("duplicate notifications: %d", n)
|
||||
}
|
||||
// The attended room is idle now; the client leaving releases it.
|
||||
again.CloseNow()
|
||||
waitFor(t, "idle room released", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
}
|
||||
|
||||
// An idle session is not observed once its last client leaves, and a server
|
||||
// start does not observe sessions whose harness reports no running turn.
|
||||
func TestChatIdleSessionWithNoClientIsNotObserved(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
store := unattendedStore(t, true)
|
||||
broker := newObservedChat()
|
||||
server := startQueueServer(t, broker, store)
|
||||
waitFor(t, "start-up probe", func() bool {
|
||||
broker.smu.Lock()
|
||||
defer broker.smu.Unlock()
|
||||
return broker.probes == 1
|
||||
})
|
||||
if _, opened := broker.streams(); opened != 0 {
|
||||
t.Fatalf("idle session observed at start: %d streams", opened)
|
||||
}
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
writeChat(t, server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
|
||||
readUntil(t, server.ctx, conn, protocol.ChatSubscribe)
|
||||
waitFor(t, "stream", func() bool { active, _ := broker.streams(); return active == 1 })
|
||||
conn.CloseNow()
|
||||
waitFor(t, "idle stream released", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
}
|
||||
|
||||
// A turn a previous server process left running is observed again at start,
|
||||
// with no client, so its reply still notifies; then the observer goes away.
|
||||
func TestChatRunningTurnObservedAfterRestart(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
store := unattendedStore(t, true)
|
||||
broker := newObservedChat()
|
||||
broker.probe["ses_1"] = true
|
||||
startQueueServer(t, broker, store)
|
||||
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
|
||||
// The harness re-announces the running turn on attach, then finishes it.
|
||||
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_9"))
|
||||
broker.push(textFrame("Finished after the restart."))
|
||||
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_9"))
|
||||
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
|
||||
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
}
|
||||
|
||||
// A recovered session whose stream never reports the turn is released after
|
||||
// the settle window rather than observed forever.
|
||||
func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
store := unattendedStore(t, true)
|
||||
broker := newObservedChat()
|
||||
broker.probe["ses_1"] = true
|
||||
startQueueServer(t, broker, store)
|
||||
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
|
||||
waitFor(t, "released after settle", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
}
|
||||
|
||||
// A turn that never reports its end does not hold an observer forever.
|
||||
func TestChatStuckUnattendedTurnIsReleased(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
limit := chatUnattendedTurnLimit
|
||||
chatUnattendedTurnLimit = 300 * time.Millisecond
|
||||
t.Cleanup(func() { chatUnattendedTurnLimit = limit })
|
||||
store := unattendedStore(t, true)
|
||||
broker := newObservedChat()
|
||||
server := startQueueServer(t, broker, store)
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
||||
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
||||
conn.CloseNow()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if active, _ := broker.streams(); active != 1 {
|
||||
t.Fatalf("in-flight turn not observed: %d", active)
|
||||
}
|
||||
waitFor(t, "stuck observer released", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
}
|
||||
|
||||
// At most chatUnattendedLimit chats are kept observing with no client.
|
||||
func TestChatUnattendedRoomsAreBounded(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
h := newChatHub()
|
||||
defer func() {
|
||||
h.mu.Lock()
|
||||
for _, room := range h.rooms {
|
||||
room.cancel()
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}()
|
||||
kept := 0
|
||||
for i := 0; i < chatUnattendedLimit+3; i++ {
|
||||
id := "ses_" + string(rune('a'+i))
|
||||
h.mu.Lock()
|
||||
room := h.roomLocked(id)
|
||||
room.running = true
|
||||
sub := &chatSub{hub: h, room: room, clientID: "c", ch: make(chan protocol.Message, 1), done: make(chan struct{})}
|
||||
room.subs["c"] = sub
|
||||
h.mu.Unlock()
|
||||
h.detach(id, sub)
|
||||
h.mu.Lock()
|
||||
if h.rooms[id] == room {
|
||||
kept++
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
if kept != chatUnattendedLimit {
|
||||
t.Fatalf("kept %d unattended rooms, want %d", kept, chatUnattendedLimit)
|
||||
}
|
||||
}
|
||||
@@ -115,6 +115,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
settingsStore = stores[0]
|
||||
}
|
||||
hub.settings = settingsStore
|
||||
if options.Chat != nil && registry != nil {
|
||||
// Keep observing chats whose turn a previous process left running, so
|
||||
// their reply or question still notifies with no client attached.
|
||||
go hub.observeRunningTurns(ctx, options.Chat)
|
||||
}
|
||||
if len(stores) > 0 {
|
||||
// Prompts queued behind a running turn are durable and owned by their
|
||||
// session; resume delivering any a previous process left queued.
|
||||
|
||||
Reference in New Issue
Block a user