Keep observing chats with an in-flight turn after the last client leaves

Chat attention notifications (questions, approvals, finished replies) come
from a session's event stream, which the hub only read while a room existed;
detach destroyed the room when the last client left, so a turn that finished
or asked with the app closed notified nobody.

A room whose last client leaves now stays observing, unattended, while its
turn is in flight or it holds an unanswered question/approval, and is torn
down when the turn ends (or after 6h for a stuck turn); at most 16 such rooms.
At server start, recently used open sessions whose harness reports a running
turn (OpenCode /api/session/active, Codex thread/turns/list) are observed
again so their reply still notifies. Settings, dedupe and app_notifications
recording are unchanged.
This commit is contained in:
2026-11-02 16:07:42 +00:00
parent 467f3eafa7
commit 80022ea41a
13 changed files with 820 additions and 25 deletions
+17 -3
View File
@@ -21,9 +21,23 @@ The same settings decide what is sent on both paths.
Each event notifies once. Opening or dismissing it on one device clears it on
the others.
Chat events are seen while some client has the chat open or the server still
holds queued messages for it. A turn that finishes with no client attached and
nothing queued does not produce a "replied" notification.
Chat events do not need the chat to be open. When the last app or browser
leaves a chat whose turn is still running, or which has an unanswered question
or approval, the server keeps watching it until the turn ends and nothing is
left to answer, so "Chat needs you" and "Chat replied" still arrive with the
app closed. A client that reopens the chat meanwhile joins the same watch, so
nothing is announced twice. Idle chats are not watched.
Limits:
- At most 16 chats are watched with no client attached; a further one is not
(queued messages are always delivered regardless).
- A turn that never reports its end stops being watched after 6 hours.
- After a server restart, the 32 most recently used open chats (used in the last
24 hours) are asked whether a turn is still running, and those that are are
watched again. This works for OpenCode and Codex, which keep running across a
Pipelines restart. A Claude Code turn runs inside the server process and ends
with it, so there is nothing to resume.
## Settings
+30
View File
@@ -0,0 +1,30 @@
package codex
import (
"context"
"encoding/json"
"sync/atomic"
"testing"
)
// ProbeTurn reports the latest turn's in-progress state from the App Server.
func TestProbeTurnReadsLatestTurn(t *testing.T) {
var completed atomic.Bool
c := modelTestChat(t, func(method string, _ json.RawMessage) any {
if method != "thread/turns/list" {
t.Errorf("unexpected method: %s", method)
}
status := "inProgress"
if completed.Load() {
status = "completed"
}
return json.RawMessage(`{"data":[{"id":"turn","status":"` + status + `","items":[]}],"nextCursor":null}`)
})
if running, err := c.ProbeTurn(context.Background(), fixtureThread); err != nil || !running {
t.Fatalf("in-progress turn: %v %v", running, err)
}
completed.Store(true)
if running, err := c.ProbeTurn(context.Background(), fixtureThread); err != nil || running {
t.Fatalf("completed turn: %v %v", running, err)
}
}
+11
View File
@@ -121,3 +121,14 @@ func (c *Chat) streamWithReadback(ctx context.Context, id string, ready func(),
}
}
}
// ProbeTurn reports whether the thread's latest turn is still in progress in
// the App Server, which outlives a Pipelines restart. The server uses it once
// at start to resume observing a turn a previous process left running.
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
last, err := c.latestTurn(ctx, id)
if err != nil {
return false, err
}
return activeTurn(last.turn), nil
}
+22
View File
@@ -1863,3 +1863,25 @@ func decodeChatSession(data []byte) (protocol.ChatSession, error) {
}
return chatSessionFields(record), nil
}
// ProbeTurn reports whether OpenCode is running a turn for the session, read
// from the same /api/session/active list Stream polls. The server uses it once
// at start to resume observing turns a previous process left running. A failed
// read is an error, never "idle".
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
if !sessionID.MatchString(id) {
return false, fmt.Errorf("invalid OpenCode session id")
}
data, err := c.call(ctx, "get", "/api/session/active", nil)
if err != nil {
return false, err
}
var response struct {
Data *map[string]json.RawMessage `json:"data"`
}
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
return false, fmt.Errorf("invalid OpenCode active-session readback")
}
_, active := (*response.Data)[id]
return active, nil
}
+44
View File
@@ -0,0 +1,44 @@
package opencode
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
)
// ProbeTurn reads the active-session list; a failed or malformed read is an
// error, never "idle".
func TestChatProbeTurn(t *testing.T) {
var malformed atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/session/active" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/json")
if malformed.Load() {
fmt.Fprint(w, `{"data":null}`)
return
}
fmt.Fprint(w, `{"data":{"ses_1":{"type":"running"}}}`)
}))
defer srv.Close()
chat := NewChat(&Connection{Backend: "v2", URL: srv.URL})
ctx := context.Background()
if running, err := chat.ProbeTurn(ctx, "ses_1"); err != nil || !running {
t.Fatalf("active session: %v %v", running, err)
}
if running, err := chat.ProbeTurn(ctx, "ses_2"); err != nil || running {
t.Fatalf("idle session: %v %v", running, err)
}
malformed.Store(true)
if _, err := chat.ProbeTurn(ctx, "ses_1"); err == nil {
t.Fatal("malformed readback treated as idle")
}
if _, err := chat.ProbeTurn(ctx, "bad id"); err == nil {
t.Fatal("invalid id accepted")
}
}
+8
View File
@@ -100,6 +100,14 @@ func TestCompactionTransportLossAndInterruptClearDoNotReleaseTurn(t *testing.T)
room.cancel()
h.drainStream(room, result)
}
if mode == "detach" {
// The last client leaving a running turn keeps the room observing
// unattended: the turn, and its live compaction, are still current.
if h.rooms[room.sessionID] != room || !room.watching || !room.running {
t.Fatal("detach released the room of a live turn")
}
return
}
if h.turns[room.sessionID].compaction != nil || !room.running {
t.Fatal("cleanup stuck label or released live turn")
}
+4 -1
View File
@@ -58,6 +58,9 @@ type chatHub struct {
// operation ids, so a client retry is acknowledged instead of re-sent.
recentOps map[string]struct{}
recentOpOrder []string
// unattended holds the unattended-observation timings (chat_unattended.go),
// fixed when the hub is built.
unattended chatUnattendedTimings
}
// chatPart is the retained form of one part. Text and reasoning deltas append to
@@ -121,7 +124,7 @@ type chatResolution struct {
}
func newChatHub() *chatHub {
return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now}
return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now, unattended: currentChatUnattendedTimings()}
}
// nextUserMessageID mints a stable id for one echoed user turn.
+16
View File
@@ -701,3 +701,19 @@ func (m *chatManager) TurnRunning(sessionID string) bool {
reporter, ok := adapter.(chatTurnReporter)
return ok && reporter.TurnRunning(sessionID)
}
// ProbeTurn asks the session's own harness whether a turn is in flight. It is
// the server-start probe (chat_unattended.go): a harness that can be asked
// (OpenCode, Codex) is asked; otherwise its in-memory state is used, which for
// Claude Code (a child of this server) is empty after a restart.
func (m *chatManager) ProbeTurn(ctx context.Context, sessionID string) (bool, error) {
adapter, err := m.brokerFor(ctx, sessionID)
if err != nil {
return false, err
}
if prober, ok := adapter.(chatTurnProber); ok {
return prober.ProbeTurn(ctx, sessionID)
}
reporter, ok := adapter.(chatTurnReporter)
return ok && reporter.TurnRunning(sessionID), nil
}
+34 -12
View File
@@ -167,6 +167,14 @@ type chatRoom struct {
// notBefore holds back a recovered queue until the restarted stream had a
// chance to report a turn the harness is still running.
notBefore time.Time
// watching is set while the unattended watcher (chat_unattended.go) owns
// this room's lifetime: the room has no client but is kept observing so a
// finished turn or a question still notifies. unattendedSince bounds how
// long a stuck turn may hold it; holdUntil keeps a room recovered at start
// alive until its stream has had a chance to report the running turn.
watching bool
unattendedSince time.Time
holdUntil time.Time
}
func newChatRoom(base context.Context, sessionID string) *chatRoom {
@@ -309,11 +317,13 @@ func (h *chatHub) attach(sessionID, clientID, label string, broker chatBroker, w
}
// detach drops one subscription. The room (and its broker stream and queue) is
// released once its last client leaves and it has no prompt left to deliver,
// so a session with no watchers holds no goroutine. A room holding queued
// released once its last client leaves and it has nothing left to do, so an
// idle session with no watchers holds no goroutine. A room holding queued
// prompts stays (with its stream) until its runner has delivered them: the
// queue belongs to the session, not to the clients watching it. A running turn
// with nothing queued behind it needs no room; the harness finishes it alone.
// queue belongs to the session, not to the clients watching it. A room whose
// turn is still in flight, or which holds an unanswered question or approval,
// also stays (bounded, see chat_unattended.go) so the operator is still
// notified when the turn finishes or the agent asks while no app is open.
func (h *chatHub) detach(sessionID string, sub *chatSub) {
h.mu.Lock()
room := h.rooms[sessionID]
@@ -322,8 +332,8 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) {
return
}
delete(room.subs, sub.clientID)
if len(room.subs) == 0 && len(room.queue) == 0 {
h.releaseRoomLocked(room)
if len(room.subs) == 0 {
h.leaveUnattendedLocked(room)
}
h.mu.Unlock()
}
@@ -461,6 +471,11 @@ func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
}
if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded && !frame.Step.Intermediate {
h.finishTurnLocked(room)
// An unwatched room kept only for this turn is done once it ends (its
// reply attention was noted above).
if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) {
h.releaseRoomLocked(room)
}
}
// Shell replay and live lifecycle delivery must enqueue in the same order
// as folding. Otherwise an exit can overtake an older running replay.
@@ -561,10 +576,12 @@ func (h *chatHub) finishTurnLocked(room *chatRoom) {
// turnInFlight reports whether sessionID has a turn in flight, so chat_close
// knows whether to stop one through the broker interrupt before it records the
// closed state. A room exists only while a client is watching the session, and
// running is set by the turn runner for the whole turn (a prompt queued behind a
// closed state. A room exists only while a client is watching the session or
// it still has work, and running is set by the turn runner for the whole turn (a prompt queued behind a
// live turn leaves it set), so an idle or unwatched session reports false and no
// interrupt is sent. It is one read under the hub mutex and mutates nothing.
// interrupt is sent. (A room also outlives its last client while its turn is in
// flight, see chat_unattended.go.) It is one read under the hub mutex and
// mutates nothing.
func (h *chatHub) turnInFlight(sessionID string) bool {
h.mu.Lock()
defer h.mu.Unlock()
@@ -709,10 +726,15 @@ func (h *chatHub) runRoom(room *chatRoom) {
}
h.runTurn(room, p, done)
// The last client may have left while this turn ran; an unwatched room
// stays only while it still has prompts to deliver.
// stays only while it still has prompts to deliver or something the
// operator must be told about.
h.mu.Lock()
if len(room.subs) == 0 && !room.running && len(room.queue) == 0 {
h.releaseRoomLocked(room)
if len(room.subs) == 0 && len(room.queue) == 0 {
if h.keepUnattendedLocked(room, false, true) {
h.watchUnattendedLocked(room)
} else {
h.releaseRoomLocked(room)
}
}
h.mu.Unlock()
}
+24 -9
View File
@@ -770,12 +770,14 @@ func TestChatMessagesRejectsInvalidSession(t *testing.T) {
}
}
// The stream of a session whose turn is still in flight outlives the client
// (so its reply still notifies), but no goroutine outlives the server.
func TestChatDisconnectCancelsStream(t *testing.T) {
broker := scriptedChat()
broker.block = true
broker.started = make(chan struct{})
broker.done = make(chan struct{})
ctx, _, conn := dialChat(t, broker)
ctx, cancel, conn := dialChat(t, broker)
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: "r1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi"}}); err != nil {
t.Fatal(err)
}
@@ -789,12 +791,17 @@ func TestChatDisconnectCancelsStream(t *testing.T) {
t.Fatal("stream never started")
}
conn.CloseNow()
// The server's read loop returns on disconnect, canceling the connection
// context the subscription observes; no goroutine may keep running.
// The turn is still in flight, so the session stays observed unattended.
select {
case <-broker.done:
t.Fatal("chat stream of an in-flight turn was canceled on disconnect")
case <-time.After(300 * time.Millisecond):
}
cancel()
select {
case <-broker.done:
case <-time.After(3 * time.Second):
t.Fatal("chat stream was not canceled on disconnect")
t.Fatal("chat stream was not canceled on shutdown")
}
if prompts, _, _ := broker.counts(); prompts != 1 {
t.Fatalf("prompt forwarded %d times, want 1", prompts)
@@ -2205,9 +2212,9 @@ func TestChatPermissionDedupesRepeatedPolls(t *testing.T) {
}
}
// TestChatDisconnectCancelsPermissionPoller pins that the permission poller is
// canceled on disconnect (reusing the stream's context), so it cannot leak a
// goroutine after the client goes away.
// TestChatDisconnectCancelsPermissionPoller pins that the permission poller
// shares the room's context: it survives a disconnect while the turn is in
// flight and cannot leak a goroutine past the server.
func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
broker := scriptedChat()
broker.block = true
@@ -2215,7 +2222,7 @@ func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
broker.done = make(chan struct{})
broker.permBlock = true
broker.permDone = make(chan struct{})
ctx, _, conn := dialChat(t, broker)
ctx, cancel, conn := dialChat(t, broker)
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: "r1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi"}}); err != nil {
t.Fatal(err)
}
@@ -2234,10 +2241,18 @@ func TestChatDisconnectCancelsPermissionPoller(t *testing.T) {
default:
}
conn.CloseNow()
// An in-flight turn keeps its poller (an approval it raises still
// notifies); shutting the server down cancels it.
select {
case <-broker.permDone:
t.Fatal("permission poller of an in-flight turn was canceled on disconnect")
case <-time.After(300 * time.Millisecond):
}
cancel()
select {
case <-broker.permDone:
case <-time.After(3 * time.Second):
t.Fatal("permission poller was not canceled on disconnect")
t.Fatal("permission poller was not canceled on shutdown")
}
select {
case <-broker.done:
+259
View File
@@ -0,0 +1,259 @@
package server
import (
"context"
"log"
"time"
)
// Chat attention notifications (chat_question, chat_approval, chat_reply) are
// derived from a session's live event stream, and the hub only reads that
// stream while the session has a room. A room normally lives while a client
// watches the chat, but the operator relies on these notifications precisely
// when no app is open. So a room whose last client leaves stays observing,
// unattended, while its turn is in flight or it holds an unanswered question
// or approval, and is torn down once that work is done. The same unattended
// observation is started at server start for sessions whose harness reports a
// running turn, so a reply that finishes after a restart still notifies.
//
// Resources are bounded: at most chatUnattendedLimit rooms are kept without a
// client (rooms holding queued prompts are not counted: they were always kept
// until delivered), a stuck turn is released after chatUnattendedTurnLimit,
// and an idle session is never observed.
// chatUnattendedLimit bounds the rooms kept observing with no client attached
// and no queued prompt.
const chatUnattendedLimit = 16
// chatRecoverSessionLimit bounds how many recently used open sessions are
// probed for a running turn at server start.
const chatRecoverSessionLimit = 32
var (
// chatUnattendedCheck is how often an unattended room re-checks whether it
// still has a reason to observe its session.
chatUnattendedCheck = 5 * time.Second
// chatUnattendedTurnLimit releases an unattended room whose turn never
// reports an end (a stuck harness or a lost end event).
chatUnattendedTurnLimit = 6 * time.Hour
// chatUnattendedRecoverSettle keeps a room recovered at start alive until
// its freshly attached stream has had time to announce the running turn.
chatUnattendedRecoverSettle = 15 * time.Second
// chatRecoverWindow limits start-up probing to sessions used recently.
chatRecoverWindow = 24 * time.Hour
)
// chatUnattendedTimings is the hub's copy of the timings above, taken when the
// hub is built so its goroutines never read the package variables.
type chatUnattendedTimings struct {
check, turnLimit, settle, window time.Duration
}
func currentChatUnattendedTimings() chatUnattendedTimings {
return chatUnattendedTimings{check: chatUnattendedCheck, turnLimit: chatUnattendedTurnLimit, settle: chatUnattendedRecoverSettle, window: chatRecoverWindow}
}
// chatTurnProber is implemented by a broker that can ask the harness itself,
// with I/O, whether a session's turn is in flight. It is used once at server
// start, when the in-memory turn state is empty.
type chatTurnProber interface {
ProbeTurn(ctx context.Context, sessionID string) (bool, error)
}
// keepUnattendedLocked reports whether a room with no client still has work
// the operator must hear about: a turn in flight (the runner's, or one its
// stream announced), an unresolved approval or question, or a harness that
// reports its own turn running (busy, read by the caller off the hub mutex).
// hold honours the start-up settle window. Queued prompts are the caller's
// concern. Callers hold h.mu.
func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool {
if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 {
return true
}
return hold && h.clock().Before(room.holdUntil)
}
// unattendedCountLocked counts the rooms kept observing with no client and no
// queued prompt, other than except. Callers hold h.mu.
func (h *chatHub) unattendedCountLocked(except *chatRoom) int {
n := 0
for _, room := range h.rooms {
if room != except && room.watching && len(room.subs) == 0 && len(room.queue) == 0 {
n++
}
}
return n
}
// leaveUnattendedLocked decides the fate of a room whose last client just
// left: a room holding queued prompts stays for its runner (as before); a room
// with an in-flight turn or pending attention keeps observing unattended,
// within the bound; anything else is released. Callers hold h.mu.
func (h *chatHub) leaveUnattendedLocked(room *chatRoom) {
if len(room.queue) > 0 {
h.watchUnattendedLocked(room)
return
}
if !h.keepUnattendedLocked(room, false, true) {
h.releaseRoomLocked(room)
return
}
if !room.watching && h.unattendedCountLocked(room) >= chatUnattendedLimit {
log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit)
h.releaseRoomLocked(room)
return
}
h.watchUnattendedLocked(room)
}
// watchUnattendedLocked starts the room's unattended watcher once. Callers
// hold h.mu.
func (h *chatHub) watchUnattendedLocked(room *chatRoom) {
if room.watching || room.ctx.Err() != nil {
return
}
room.watching = true
room.unattendedSince = h.clock()
go h.watchUnattended(room)
}
func (h *chatHub) watchUnattended(room *chatRoom) {
check := h.unattended.check
if check <= 0 {
check = 5 * time.Second
}
ticker := time.NewTicker(check)
defer ticker.Stop()
for {
select {
case <-room.ctx.Done():
return
case <-ticker.C:
}
if h.reapUnattended(room) {
return
}
}
}
// reapUnattended re-checks one unattended room and reports whether its
// watcher is finished: a client reattached (the room is attended again), or
// the room had nothing left to observe, or its turn exceeded the limit, and
// was released.
func (h *chatHub) reapUnattended(room *chatRoom) bool {
h.mu.Lock()
if done := h.attendedLocked(room); done {
h.mu.Unlock()
return true
}
broker := room.broker
h.mu.Unlock()
// TurnRunning performs no I/O but may take the broker's own lock, so it is
// never asked under the hub mutex.
busy := brokerTurnRunning(broker, room.sessionID)
h.mu.Lock()
defer h.mu.Unlock()
if done := h.attendedLocked(room); done {
return true
}
if len(room.queue) > 0 {
// The runner owns a room with queued prompts until they are delivered;
// the stuck-turn limit counts from the last of them.
room.unattendedSince = h.clock()
return false
}
expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit
if !expired && h.keepUnattendedLocked(room, busy, true) {
return false
}
if expired {
log.Printf("chat session %s: released the unattended observer after %s without the turn ending", room.sessionID, h.unattended.turnLimit)
}
room.watching = false
h.releaseRoomLocked(room)
return true
}
// attendedLocked reports whether the watcher should stop because the room is
// gone or a client is watching it again. Callers hold h.mu.
func (h *chatHub) attendedLocked(room *chatRoom) bool {
if h.rooms[room.sessionID] != room || room.ctx.Err() != nil {
room.watching = false
return true
}
if len(room.subs) > 0 {
room.watching = false
room.unattendedSince = time.Time{}
return true
}
return false
}
// observeRunningTurns is the start-up half: each recently used open operator
// session whose harness reports a turn in flight gets an unattended room
// observing its stream, so the reply (or a question) still notifies. It is
// best effort per harness: OpenCode and Codex keep running across a Pipelines
// restart and report it; Claude Code runs as a child of the server and so has
// no turn left after a restart.
func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
prober, ok := broker.(chatTurnProber)
if !ok || h.registry == nil {
return
}
sessions, err := h.registry.OperatorSessions(ctx)
if err != nil {
log.Printf("chat turn recovery: %v", err)
return
}
cutoff := h.clock().Add(-h.unattended.window).UnixMilli()
probed := 0
for _, session := range sessions {
if ctx.Err() != nil {
return
}
if session.ClosedAt != 0 || session.UpdatedAt < cutoff || !validChatSessionID(session.ID) {
continue
}
if probed >= chatRecoverSessionLimit {
break
}
probed++
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
running, err := prober.ProbeTurn(probeCtx, session.ID)
cancel()
if err != nil {
log.Printf("chat turn recovery probe session=%s failed (%T)", session.ID, err)
continue
}
if running {
h.observeUnattended(session.ID, broker)
}
}
}
// observeUnattended opens (or reuses) the session's room with no client and
// observes its stream until the turn the harness reported has ended.
func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
h.mu.Lock()
if h.rooms[sessionID] == nil && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
h.mu.Unlock()
log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit)
return
}
room := h.roomLocked(sessionID)
if room.broker == nil {
room.broker = broker
}
if len(room.subs) == 0 {
room.holdUntil = h.clock().Add(h.unattended.settle)
h.watchUnattendedLocked(room)
}
h.mu.Unlock()
if err := h.ensureObserving(room, broker); err != nil {
if room.ctx.Err() == nil {
log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err)
}
return
}
h.startRoomPoller(room, broker)
}
+346
View File
@@ -0,0 +1,346 @@
package server
import (
"context"
"path/filepath"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// observedChat is a channelChat whose stream can be opened more than once and
// reports how many streams are live, so a test can prove when the hub observes
// a session with no client attached and when it stops. probe answers the
// server-start ProbeTurn.
type observedChat struct {
*channelChat
smu sync.Mutex
active int
opened int
probe map[string]bool
probes int
}
func newObservedChat() *observedChat {
return &observedChat{channelChat: newChannelChat(), probe: map[string]bool{}}
}
func (c *observedChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
c.smu.Lock()
c.active++
c.opened++
c.smu.Unlock()
defer func() {
c.smu.Lock()
c.active--
c.smu.Unlock()
}()
if ready != nil {
ready()
}
for {
select {
case <-ctx.Done():
return ctx.Err()
case frame := <-c.frames:
if err := emit(frame); err != nil {
return err
}
}
}
}
func (c *observedChat) ProbeTurn(_ context.Context, sessionID string) (bool, error) {
c.smu.Lock()
defer c.smu.Unlock()
c.probes++
return c.probe[sessionID], nil
}
func (c *observedChat) streams() (active, opened int) {
c.smu.Lock()
defer c.smu.Unlock()
return c.active, c.opened
}
func fastUnattended(t *testing.T) {
t.Helper()
check, settle := chatUnattendedCheck, chatUnattendedRecoverSettle
chatUnattendedCheck = 10 * time.Millisecond
chatUnattendedRecoverSettle = 200 * time.Millisecond
t.Cleanup(func() { chatUnattendedCheck, chatUnattendedRecoverSettle = check, settle })
}
func unattendedStore(t *testing.T, replies bool) *pipeline.Store {
t.Helper()
store := openQueueStore(t, filepath.Join(t.TempDir(), "chat.db"))
if err := store.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
if replies {
if _, err := store.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
t.Fatal(err)
}
}
return store
}
// chatNotes returns the recorded app notifications of one kind.
func chatNotes(t *testing.T, store *pipeline.Store, kind string) []protocol.AppNotification {
t.Helper()
active, _, err := store.ActiveAppNotifications(context.Background(), "", time.Now())
if err != nil {
t.Fatal(err)
}
var out []protocol.AppNotification
for _, n := range active {
if n.Kind == kind {
out = append(out, n)
}
}
return out
}
func stepFrameFor(status, turn string) protocol.ChatEventFrame {
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: status, TurnID: turn}}
}
func textFrame(text string) protocol.ChatEventFrame {
return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventText, MessageID: "msg_a", Part: &protocol.ChatPart{ID: "msg_a_p"}, Delta: text}
}
// A turn started from the app keeps being observed after the app closes, so
// its finished reply is recorded (app stream) and published (ntfy) once, and
// the observer is torn down when the turn ends.
func TestChatReplyNotifiesAfterLastClientLeaves(t *testing.T) {
fastUnattended(t)
captured := newNtfyCapture(t)
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
waitFor(t, "prompt", func() bool { return broker.promptCount() == 1 })
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
broker.push(textFrame("The build is green."))
conn.CloseNow()
time.Sleep(100 * time.Millisecond) // let the server notice the disconnect
if active, _ := broker.streams(); active != 1 {
t.Fatalf("in-flight turn not observed after the client left: %d streams", active)
}
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
waitFor(t, "ntfy publish", func() bool { return len(captured()) == 1 })
time.Sleep(50 * time.Millisecond)
notes := chatNotes(t, store, protocol.NotificationChatReply)
if got := captured(); len(notes) != 1 || len(got) != 1 || got[0].Title != "Release prep replied" || got[0].Message != "The build is green." {
t.Fatalf("notes=%+v ntfy=%+v", notes, got)
}
if notes[0].Message != "The build is green." || notes[0].Click != "pipelines://chat/ses_1" {
t.Fatalf("note = %+v", notes[0])
}
}
// The chat_reply_finished setting still decides: off (the default) records
// nothing, even though the unattended turn is observed.
func TestChatUnattendedReplyRespectsSetting(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, false)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
time.Sleep(100 * time.Millisecond)
if notes := chatNotes(t, store, protocol.NotificationChatReply); len(notes) != 0 {
t.Fatalf("reply recorded with chat_reply_finished off: %+v", notes)
}
}
// A question the agent asks while no client is attached notifies.
func TestChatQuestionNotifiesWithNoClient(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, false)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "ship?", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Which branch should ship?"}
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
if notes := chatNotes(t, store, protocol.NotificationChatQuestion); notes[0].Message != "Which branch should ship?" {
t.Fatalf("note = %+v", notes[0])
}
// The turn ends with the question withdrawn: nothing is left to observe.
form.Cancelled = true
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A client reattaching mid-turn joins the observing room: the reply is still
// announced exactly once and the client sees the turn end.
func TestChatReattachMidTurnDoesNotDuplicate(t *testing.T) {
fastUnattended(t)
captured := newNtfyCapture(t)
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
broker.push(textFrame("Done."))
form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Continue?"}
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 })
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
again := dialChatConn(t, server.ctx, server.srv)
writeChat(t, server.ctx, again, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, server.ctx, again, protocol.ChatSubscribe)
if _, opened := broker.streams(); opened != 1 {
t.Fatalf("reattach opened a second stream: %d", opened)
}
form.Resolved = true
broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form})
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
for {
msg := readUntil(t, server.ctx, again, protocol.ChatEvent)
if e := msg.ChatEvent; e.Kind == protocol.ChatEventStep && e.Step != nil && e.Step.Status == protocol.ChatStepEnded {
break
}
}
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
time.Sleep(100 * time.Millisecond)
if got := captured(); len(got) != 2 {
t.Fatalf("ntfy deliveries = %+v, want one question and one reply", got)
}
if n := len(chatNotes(t, store, protocol.NotificationChatQuestion)) + len(chatNotes(t, store, protocol.NotificationChatReply)); n > 2 {
t.Fatalf("duplicate notifications: %d", n)
}
// The attended room is idle now; the client leaving releases it.
again.CloseNow()
waitFor(t, "idle room released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// An idle session is not observed once its last client leaves, and a server
// start does not observe sessions whose harness reports no running turn.
func TestChatIdleSessionWithNoClientIsNotObserved(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
waitFor(t, "start-up probe", func() bool {
broker.smu.Lock()
defer broker.smu.Unlock()
return broker.probes == 1
})
if _, opened := broker.streams(); opened != 0 {
t.Fatalf("idle session observed at start: %d streams", opened)
}
conn := dialChatConn(t, server.ctx, server.srv)
writeChat(t, server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, server.ctx, conn, protocol.ChatSubscribe)
waitFor(t, "stream", func() bool { active, _ := broker.streams(); return active == 1 })
conn.CloseNow()
waitFor(t, "idle stream released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A turn a previous server process left running is observed again at start,
// with no client, so its reply still notifies; then the observer goes away.
func TestChatRunningTurnObservedAfterRestart(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
broker.probe["ses_1"] = true
startQueueServer(t, broker, store)
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
// The harness re-announces the running turn on attach, then finishes it.
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_9"))
broker.push(textFrame("Finished after the restart."))
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_9"))
waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 })
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A recovered session whose stream never reports the turn is released after
// the settle window rather than observed forever.
func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
broker := newObservedChat()
broker.probe["ses_1"] = true
startQueueServer(t, broker, store)
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
waitFor(t, "released after settle", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A turn that never reports its end does not hold an observer forever.
func TestChatStuckUnattendedTurnIsReleased(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
limit := chatUnattendedTurnLimit
chatUnattendedTurnLimit = 300 * time.Millisecond
t.Cleanup(func() { chatUnattendedTurnLimit = limit })
store := unattendedStore(t, true)
broker := newObservedChat()
server := startQueueServer(t, broker, store)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
conn.CloseNow()
time.Sleep(100 * time.Millisecond)
if active, _ := broker.streams(); active != 1 {
t.Fatalf("in-flight turn not observed: %d", active)
}
waitFor(t, "stuck observer released", func() bool { active, _ := broker.streams(); return active == 0 })
}
// At most chatUnattendedLimit chats are kept observing with no client.
func TestChatUnattendedRoomsAreBounded(t *testing.T) {
fastUnattended(t)
h := newChatHub()
defer func() {
h.mu.Lock()
for _, room := range h.rooms {
room.cancel()
}
h.mu.Unlock()
}()
kept := 0
for i := 0; i < chatUnattendedLimit+3; i++ {
id := "ses_" + string(rune('a'+i))
h.mu.Lock()
room := h.roomLocked(id)
room.running = true
sub := &chatSub{hub: h, room: room, clientID: "c", ch: make(chan protocol.Message, 1), done: make(chan struct{})}
room.subs["c"] = sub
h.mu.Unlock()
h.detach(id, sub)
h.mu.Lock()
if h.rooms[id] == room {
kept++
}
h.mu.Unlock()
}
if kept != chatUnattendedLimit {
t.Fatalf("kept %d unattended rooms, want %d", kept, chatUnattendedLimit)
}
}
+5
View File
@@ -115,6 +115,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
settingsStore = stores[0]
}
hub.settings = settingsStore
if options.Chat != nil && registry != nil {
// Keep observing chats whose turn a previous process left running, so
// their reply or question still notifies with no client attached.
go hub.observeRunningTurns(ctx, options.Chat)
}
if len(stores) > 0 {
// Prompts queued behind a running turn are durable and owned by their
// session; resume delivering any a previous process left queued.