From 80022ea41a347852fbfbeb6aca541ddd579e83e1 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Mon, 2 Nov 2026 16:07:42 +0000 Subject: [PATCH] Keep observing chats with an in-flight turn after the last client leaves Chat attention notifications (questions, approvals, finished replies) come from a session's event stream, which the hub only read while a room existed; detach destroyed the room when the last client left, so a turn that finished or asked with the app closed notified nobody. A room whose last client leaves now stays observing, unattended, while its turn is in flight or it holds an unanswered question/approval, and is torn down when the turn ends (or after 6h for a stuck turn); at most 16 such rooms. At server start, recently used open sessions whose harness reports a running turn (OpenCode /api/session/active, Codex thread/turns/list) are observed again so their reply still notifies. Settings, dedupe and app_notifications recording are unchanged. --- docs/notifications.md | 20 +- internal/codex/probe_test.go | 30 ++ internal/codex/readback.go | 11 + internal/opencode/chat.go | 22 ++ internal/opencode/chat_probe_test.go | 44 +++ internal/server/chat_compaction_test.go | 8 + internal/server/chat_hub.go | 5 +- internal/server/chat_manager.go | 16 ++ internal/server/chat_room.go | 46 +++- internal/server/chat_test.go | 33 ++- internal/server/chat_unattended.go | 259 ++++++++++++++++++ internal/server/chat_unattended_test.go | 346 ++++++++++++++++++++++++ internal/server/websocket.go | 5 + 13 files changed, 820 insertions(+), 25 deletions(-) create mode 100644 internal/codex/probe_test.go create mode 100644 internal/opencode/chat_probe_test.go create mode 100644 internal/server/chat_unattended.go create mode 100644 internal/server/chat_unattended_test.go diff --git a/docs/notifications.md b/docs/notifications.md index 479b804a..ee71b646 100644 --- a/docs/notifications.md +++ b/docs/notifications.md @@ -21,9 +21,23 @@ The same settings decide what is sent on both paths. Each event notifies once. Opening or dismissing it on one device clears it on the others. -Chat events are seen while some client has the chat open or the server still -holds queued messages for it. A turn that finishes with no client attached and -nothing queued does not produce a "replied" notification. +Chat events do not need the chat to be open. When the last app or browser +leaves a chat whose turn is still running, or which has an unanswered question +or approval, the server keeps watching it until the turn ends and nothing is +left to answer, so "Chat needs you" and "Chat replied" still arrive with the +app closed. A client that reopens the chat meanwhile joins the same watch, so +nothing is announced twice. Idle chats are not watched. + +Limits: + +- At most 16 chats are watched with no client attached; a further one is not + (queued messages are always delivered regardless). +- A turn that never reports its end stops being watched after 6 hours. +- After a server restart, the 32 most recently used open chats (used in the last + 24 hours) are asked whether a turn is still running, and those that are are + watched again. This works for OpenCode and Codex, which keep running across a + Pipelines restart. A Claude Code turn runs inside the server process and ends + with it, so there is nothing to resume. ## Settings diff --git a/internal/codex/probe_test.go b/internal/codex/probe_test.go new file mode 100644 index 00000000..c6503642 --- /dev/null +++ b/internal/codex/probe_test.go @@ -0,0 +1,30 @@ +package codex + +import ( + "context" + "encoding/json" + "sync/atomic" + "testing" +) + +// ProbeTurn reports the latest turn's in-progress state from the App Server. +func TestProbeTurnReadsLatestTurn(t *testing.T) { + var completed atomic.Bool + c := modelTestChat(t, func(method string, _ json.RawMessage) any { + if method != "thread/turns/list" { + t.Errorf("unexpected method: %s", method) + } + status := "inProgress" + if completed.Load() { + status = "completed" + } + return json.RawMessage(`{"data":[{"id":"turn","status":"` + status + `","items":[]}],"nextCursor":null}`) + }) + if running, err := c.ProbeTurn(context.Background(), fixtureThread); err != nil || !running { + t.Fatalf("in-progress turn: %v %v", running, err) + } + completed.Store(true) + if running, err := c.ProbeTurn(context.Background(), fixtureThread); err != nil || running { + t.Fatalf("completed turn: %v %v", running, err) + } +} diff --git a/internal/codex/readback.go b/internal/codex/readback.go index 3b843213..7b7e919a 100644 --- a/internal/codex/readback.go +++ b/internal/codex/readback.go @@ -121,3 +121,14 @@ func (c *Chat) streamWithReadback(ctx context.Context, id string, ready func(), } } } + +// ProbeTurn reports whether the thread's latest turn is still in progress in +// the App Server, which outlives a Pipelines restart. The server uses it once +// at start to resume observing a turn a previous process left running. +func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) { + last, err := c.latestTurn(ctx, id) + if err != nil { + return false, err + } + return activeTurn(last.turn), nil +} diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index 64095555..c66f547e 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -1863,3 +1863,25 @@ func decodeChatSession(data []byte) (protocol.ChatSession, error) { } return chatSessionFields(record), nil } + +// ProbeTurn reports whether OpenCode is running a turn for the session, read +// from the same /api/session/active list Stream polls. The server uses it once +// at start to resume observing turns a previous process left running. A failed +// read is an error, never "idle". +func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) { + if !sessionID.MatchString(id) { + return false, fmt.Errorf("invalid OpenCode session id") + } + data, err := c.call(ctx, "get", "/api/session/active", nil) + if err != nil { + return false, err + } + var response struct { + Data *map[string]json.RawMessage `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || response.Data == nil { + return false, fmt.Errorf("invalid OpenCode active-session readback") + } + _, active := (*response.Data)[id] + return active, nil +} diff --git a/internal/opencode/chat_probe_test.go b/internal/opencode/chat_probe_test.go new file mode 100644 index 00000000..8b4c49e1 --- /dev/null +++ b/internal/opencode/chat_probe_test.go @@ -0,0 +1,44 @@ +package opencode + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +// ProbeTurn reads the active-session list; a failed or malformed read is an +// error, never "idle". +func TestChatProbeTurn(t *testing.T) { + var malformed atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/session/active" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + if malformed.Load() { + fmt.Fprint(w, `{"data":null}`) + return + } + fmt.Fprint(w, `{"data":{"ses_1":{"type":"running"}}}`) + })) + defer srv.Close() + chat := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + ctx := context.Background() + if running, err := chat.ProbeTurn(ctx, "ses_1"); err != nil || !running { + t.Fatalf("active session: %v %v", running, err) + } + if running, err := chat.ProbeTurn(ctx, "ses_2"); err != nil || running { + t.Fatalf("idle session: %v %v", running, err) + } + malformed.Store(true) + if _, err := chat.ProbeTurn(ctx, "ses_1"); err == nil { + t.Fatal("malformed readback treated as idle") + } + if _, err := chat.ProbeTurn(ctx, "bad id"); err == nil { + t.Fatal("invalid id accepted") + } +} diff --git a/internal/server/chat_compaction_test.go b/internal/server/chat_compaction_test.go index 3741f0c9..63ddcab7 100644 --- a/internal/server/chat_compaction_test.go +++ b/internal/server/chat_compaction_test.go @@ -100,6 +100,14 @@ func TestCompactionTransportLossAndInterruptClearDoNotReleaseTurn(t *testing.T) room.cancel() h.drainStream(room, result) } + if mode == "detach" { + // The last client leaving a running turn keeps the room observing + // unattended: the turn, and its live compaction, are still current. + if h.rooms[room.sessionID] != room || !room.watching || !room.running { + t.Fatal("detach released the room of a live turn") + } + return + } if h.turns[room.sessionID].compaction != nil || !room.running { t.Fatal("cleanup stuck label or released live turn") } diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 1f8a220d..662fb4e7 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -58,6 +58,9 @@ type chatHub struct { // operation ids, so a client retry is acknowledged instead of re-sent. recentOps map[string]struct{} recentOpOrder []string + // unattended holds the unattended-observation timings (chat_unattended.go), + // fixed when the hub is built. + unattended chatUnattendedTimings } // chatPart is the retained form of one part. Text and reasoning deltas append to @@ -121,7 +124,7 @@ type chatResolution struct { } func newChatHub() *chatHub { - return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now} + return &chatHub{turns: map[string]*chatTurn{}, rooms: map[string]*chatRoom{}, baseCtx: context.Background(), now: time.Now, unattended: currentChatUnattendedTimings()} } // nextUserMessageID mints a stable id for one echoed user turn. diff --git a/internal/server/chat_manager.go b/internal/server/chat_manager.go index 3e94047a..923d6c6f 100644 --- a/internal/server/chat_manager.go +++ b/internal/server/chat_manager.go @@ -701,3 +701,19 @@ func (m *chatManager) TurnRunning(sessionID string) bool { reporter, ok := adapter.(chatTurnReporter) return ok && reporter.TurnRunning(sessionID) } + +// ProbeTurn asks the session's own harness whether a turn is in flight. It is +// the server-start probe (chat_unattended.go): a harness that can be asked +// (OpenCode, Codex) is asked; otherwise its in-memory state is used, which for +// Claude Code (a child of this server) is empty after a restart. +func (m *chatManager) ProbeTurn(ctx context.Context, sessionID string) (bool, error) { + adapter, err := m.brokerFor(ctx, sessionID) + if err != nil { + return false, err + } + if prober, ok := adapter.(chatTurnProber); ok { + return prober.ProbeTurn(ctx, sessionID) + } + reporter, ok := adapter.(chatTurnReporter) + return ok && reporter.TurnRunning(sessionID), nil +} diff --git a/internal/server/chat_room.go b/internal/server/chat_room.go index 09e0148d..e9e69ee2 100644 --- a/internal/server/chat_room.go +++ b/internal/server/chat_room.go @@ -167,6 +167,14 @@ type chatRoom struct { // notBefore holds back a recovered queue until the restarted stream had a // chance to report a turn the harness is still running. notBefore time.Time + // watching is set while the unattended watcher (chat_unattended.go) owns + // this room's lifetime: the room has no client but is kept observing so a + // finished turn or a question still notifies. unattendedSince bounds how + // long a stuck turn may hold it; holdUntil keeps a room recovered at start + // alive until its stream has had a chance to report the running turn. + watching bool + unattendedSince time.Time + holdUntil time.Time } func newChatRoom(base context.Context, sessionID string) *chatRoom { @@ -309,11 +317,13 @@ func (h *chatHub) attach(sessionID, clientID, label string, broker chatBroker, w } // detach drops one subscription. The room (and its broker stream and queue) is -// released once its last client leaves and it has no prompt left to deliver, -// so a session with no watchers holds no goroutine. A room holding queued +// released once its last client leaves and it has nothing left to do, so an +// idle session with no watchers holds no goroutine. A room holding queued // prompts stays (with its stream) until its runner has delivered them: the -// queue belongs to the session, not to the clients watching it. A running turn -// with nothing queued behind it needs no room; the harness finishes it alone. +// queue belongs to the session, not to the clients watching it. A room whose +// turn is still in flight, or which holds an unanswered question or approval, +// also stays (bounded, see chat_unattended.go) so the operator is still +// notified when the turn finishes or the agent asks while no app is open. func (h *chatHub) detach(sessionID string, sub *chatSub) { h.mu.Lock() room := h.rooms[sessionID] @@ -322,8 +332,8 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) { return } delete(room.subs, sub.clientID) - if len(room.subs) == 0 && len(room.queue) == 0 { - h.releaseRoomLocked(room) + if len(room.subs) == 0 { + h.leaveUnattendedLocked(room) } h.mu.Unlock() } @@ -461,6 +471,11 @@ func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) { } if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded && !frame.Step.Intermediate { h.finishTurnLocked(room) + // An unwatched room kept only for this turn is done once it ends (its + // reply attention was noted above). + if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) { + h.releaseRoomLocked(room) + } } // Shell replay and live lifecycle delivery must enqueue in the same order // as folding. Otherwise an exit can overtake an older running replay. @@ -561,10 +576,12 @@ func (h *chatHub) finishTurnLocked(room *chatRoom) { // turnInFlight reports whether sessionID has a turn in flight, so chat_close // knows whether to stop one through the broker interrupt before it records the -// closed state. A room exists only while a client is watching the session, and -// running is set by the turn runner for the whole turn (a prompt queued behind a +// closed state. A room exists only while a client is watching the session or +// it still has work, and running is set by the turn runner for the whole turn (a prompt queued behind a // live turn leaves it set), so an idle or unwatched session reports false and no -// interrupt is sent. It is one read under the hub mutex and mutates nothing. +// interrupt is sent. (A room also outlives its last client while its turn is in +// flight, see chat_unattended.go.) It is one read under the hub mutex and +// mutates nothing. func (h *chatHub) turnInFlight(sessionID string) bool { h.mu.Lock() defer h.mu.Unlock() @@ -709,10 +726,15 @@ func (h *chatHub) runRoom(room *chatRoom) { } h.runTurn(room, p, done) // The last client may have left while this turn ran; an unwatched room - // stays only while it still has prompts to deliver. + // stays only while it still has prompts to deliver or something the + // operator must be told about. h.mu.Lock() - if len(room.subs) == 0 && !room.running && len(room.queue) == 0 { - h.releaseRoomLocked(room) + if len(room.subs) == 0 && len(room.queue) == 0 { + if h.keepUnattendedLocked(room, false, true) { + h.watchUnattendedLocked(room) + } else { + h.releaseRoomLocked(room) + } } h.mu.Unlock() } diff --git a/internal/server/chat_test.go b/internal/server/chat_test.go index 6dfe6efa..4c80df85 100644 --- a/internal/server/chat_test.go +++ b/internal/server/chat_test.go @@ -770,12 +770,14 @@ func TestChatMessagesRejectsInvalidSession(t *testing.T) { } } +// The stream of a session whose turn is still in flight outlives the client +// (so its reply still notifies), but no goroutine outlives the server. func TestChatDisconnectCancelsStream(t *testing.T) { broker := scriptedChat() broker.block = true broker.started = make(chan struct{}) broker.done = make(chan struct{}) - ctx, _, conn := dialChat(t, broker) + ctx, cancel, conn := dialChat(t, broker) if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: "r1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi"}}); err != nil { t.Fatal(err) } @@ -789,12 +791,17 @@ func TestChatDisconnectCancelsStream(t *testing.T) { t.Fatal("stream never started") } conn.CloseNow() - // The server's read loop returns on disconnect, canceling the connection - // context the subscription observes; no goroutine may keep running. + // The turn is still in flight, so the session stays observed unattended. + select { + case <-broker.done: + t.Fatal("chat stream of an in-flight turn was canceled on disconnect") + case <-time.After(300 * time.Millisecond): + } + cancel() select { case <-broker.done: case <-time.After(3 * time.Second): - t.Fatal("chat stream was not canceled on disconnect") + t.Fatal("chat stream was not canceled on shutdown") } if prompts, _, _ := broker.counts(); prompts != 1 { t.Fatalf("prompt forwarded %d times, want 1", prompts) @@ -2205,9 +2212,9 @@ func TestChatPermissionDedupesRepeatedPolls(t *testing.T) { } } -// TestChatDisconnectCancelsPermissionPoller pins that the permission poller is -// canceled on disconnect (reusing the stream's context), so it cannot leak a -// goroutine after the client goes away. +// TestChatDisconnectCancelsPermissionPoller pins that the permission poller +// shares the room's context: it survives a disconnect while the turn is in +// flight and cannot leak a goroutine past the server. func TestChatDisconnectCancelsPermissionPoller(t *testing.T) { broker := scriptedChat() broker.block = true @@ -2215,7 +2222,7 @@ func TestChatDisconnectCancelsPermissionPoller(t *testing.T) { broker.done = make(chan struct{}) broker.permBlock = true broker.permDone = make(chan struct{}) - ctx, _, conn := dialChat(t, broker) + ctx, cancel, conn := dialChat(t, broker) if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatPrompt, ID: "r1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: "ses_1", Text: "hi"}}); err != nil { t.Fatal(err) } @@ -2234,10 +2241,18 @@ func TestChatDisconnectCancelsPermissionPoller(t *testing.T) { default: } conn.CloseNow() + // An in-flight turn keeps its poller (an approval it raises still + // notifies); shutting the server down cancels it. + select { + case <-broker.permDone: + t.Fatal("permission poller of an in-flight turn was canceled on disconnect") + case <-time.After(300 * time.Millisecond): + } + cancel() select { case <-broker.permDone: case <-time.After(3 * time.Second): - t.Fatal("permission poller was not canceled on disconnect") + t.Fatal("permission poller was not canceled on shutdown") } select { case <-broker.done: diff --git a/internal/server/chat_unattended.go b/internal/server/chat_unattended.go new file mode 100644 index 00000000..76507903 --- /dev/null +++ b/internal/server/chat_unattended.go @@ -0,0 +1,259 @@ +package server + +import ( + "context" + "log" + "time" +) + +// Chat attention notifications (chat_question, chat_approval, chat_reply) are +// derived from a session's live event stream, and the hub only reads that +// stream while the session has a room. A room normally lives while a client +// watches the chat, but the operator relies on these notifications precisely +// when no app is open. So a room whose last client leaves stays observing, +// unattended, while its turn is in flight or it holds an unanswered question +// or approval, and is torn down once that work is done. The same unattended +// observation is started at server start for sessions whose harness reports a +// running turn, so a reply that finishes after a restart still notifies. +// +// Resources are bounded: at most chatUnattendedLimit rooms are kept without a +// client (rooms holding queued prompts are not counted: they were always kept +// until delivered), a stuck turn is released after chatUnattendedTurnLimit, +// and an idle session is never observed. + +// chatUnattendedLimit bounds the rooms kept observing with no client attached +// and no queued prompt. +const chatUnattendedLimit = 16 + +// chatRecoverSessionLimit bounds how many recently used open sessions are +// probed for a running turn at server start. +const chatRecoverSessionLimit = 32 + +var ( + // chatUnattendedCheck is how often an unattended room re-checks whether it + // still has a reason to observe its session. + chatUnattendedCheck = 5 * time.Second + // chatUnattendedTurnLimit releases an unattended room whose turn never + // reports an end (a stuck harness or a lost end event). + chatUnattendedTurnLimit = 6 * time.Hour + // chatUnattendedRecoverSettle keeps a room recovered at start alive until + // its freshly attached stream has had time to announce the running turn. + chatUnattendedRecoverSettle = 15 * time.Second + // chatRecoverWindow limits start-up probing to sessions used recently. + chatRecoverWindow = 24 * time.Hour +) + +// chatUnattendedTimings is the hub's copy of the timings above, taken when the +// hub is built so its goroutines never read the package variables. +type chatUnattendedTimings struct { + check, turnLimit, settle, window time.Duration +} + +func currentChatUnattendedTimings() chatUnattendedTimings { + return chatUnattendedTimings{check: chatUnattendedCheck, turnLimit: chatUnattendedTurnLimit, settle: chatUnattendedRecoverSettle, window: chatRecoverWindow} +} + +// chatTurnProber is implemented by a broker that can ask the harness itself, +// with I/O, whether a session's turn is in flight. It is used once at server +// start, when the in-memory turn state is empty. +type chatTurnProber interface { + ProbeTurn(ctx context.Context, sessionID string) (bool, error) +} + +// keepUnattendedLocked reports whether a room with no client still has work +// the operator must hear about: a turn in flight (the runner's, or one its +// stream announced), an unresolved approval or question, or a harness that +// reports its own turn running (busy, read by the caller off the hub mutex). +// hold honours the start-up settle window. Queued prompts are the caller's +// concern. Callers hold h.mu. +func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool { + if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 { + return true + } + return hold && h.clock().Before(room.holdUntil) +} + +// unattendedCountLocked counts the rooms kept observing with no client and no +// queued prompt, other than except. Callers hold h.mu. +func (h *chatHub) unattendedCountLocked(except *chatRoom) int { + n := 0 + for _, room := range h.rooms { + if room != except && room.watching && len(room.subs) == 0 && len(room.queue) == 0 { + n++ + } + } + return n +} + +// leaveUnattendedLocked decides the fate of a room whose last client just +// left: a room holding queued prompts stays for its runner (as before); a room +// with an in-flight turn or pending attention keeps observing unattended, +// within the bound; anything else is released. Callers hold h.mu. +func (h *chatHub) leaveUnattendedLocked(room *chatRoom) { + if len(room.queue) > 0 { + h.watchUnattendedLocked(room) + return + } + if !h.keepUnattendedLocked(room, false, true) { + h.releaseRoomLocked(room) + return + } + if !room.watching && h.unattendedCountLocked(room) >= chatUnattendedLimit { + log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit) + h.releaseRoomLocked(room) + return + } + h.watchUnattendedLocked(room) +} + +// watchUnattendedLocked starts the room's unattended watcher once. Callers +// hold h.mu. +func (h *chatHub) watchUnattendedLocked(room *chatRoom) { + if room.watching || room.ctx.Err() != nil { + return + } + room.watching = true + room.unattendedSince = h.clock() + go h.watchUnattended(room) +} + +func (h *chatHub) watchUnattended(room *chatRoom) { + check := h.unattended.check + if check <= 0 { + check = 5 * time.Second + } + ticker := time.NewTicker(check) + defer ticker.Stop() + for { + select { + case <-room.ctx.Done(): + return + case <-ticker.C: + } + if h.reapUnattended(room) { + return + } + } +} + +// reapUnattended re-checks one unattended room and reports whether its +// watcher is finished: a client reattached (the room is attended again), or +// the room had nothing left to observe, or its turn exceeded the limit, and +// was released. +func (h *chatHub) reapUnattended(room *chatRoom) bool { + h.mu.Lock() + if done := h.attendedLocked(room); done { + h.mu.Unlock() + return true + } + broker := room.broker + h.mu.Unlock() + // TurnRunning performs no I/O but may take the broker's own lock, so it is + // never asked under the hub mutex. + busy := brokerTurnRunning(broker, room.sessionID) + h.mu.Lock() + defer h.mu.Unlock() + if done := h.attendedLocked(room); done { + return true + } + if len(room.queue) > 0 { + // The runner owns a room with queued prompts until they are delivered; + // the stuck-turn limit counts from the last of them. + room.unattendedSince = h.clock() + return false + } + expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit + if !expired && h.keepUnattendedLocked(room, busy, true) { + return false + } + if expired { + log.Printf("chat session %s: released the unattended observer after %s without the turn ending", room.sessionID, h.unattended.turnLimit) + } + room.watching = false + h.releaseRoomLocked(room) + return true +} + +// attendedLocked reports whether the watcher should stop because the room is +// gone or a client is watching it again. Callers hold h.mu. +func (h *chatHub) attendedLocked(room *chatRoom) bool { + if h.rooms[room.sessionID] != room || room.ctx.Err() != nil { + room.watching = false + return true + } + if len(room.subs) > 0 { + room.watching = false + room.unattendedSince = time.Time{} + return true + } + return false +} + +// observeRunningTurns is the start-up half: each recently used open operator +// session whose harness reports a turn in flight gets an unattended room +// observing its stream, so the reply (or a question) still notifies. It is +// best effort per harness: OpenCode and Codex keep running across a Pipelines +// restart and report it; Claude Code runs as a child of the server and so has +// no turn left after a restart. +func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) { + prober, ok := broker.(chatTurnProber) + if !ok || h.registry == nil { + return + } + sessions, err := h.registry.OperatorSessions(ctx) + if err != nil { + log.Printf("chat turn recovery: %v", err) + return + } + cutoff := h.clock().Add(-h.unattended.window).UnixMilli() + probed := 0 + for _, session := range sessions { + if ctx.Err() != nil { + return + } + if session.ClosedAt != 0 || session.UpdatedAt < cutoff || !validChatSessionID(session.ID) { + continue + } + if probed >= chatRecoverSessionLimit { + break + } + probed++ + probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + running, err := prober.ProbeTurn(probeCtx, session.ID) + cancel() + if err != nil { + log.Printf("chat turn recovery probe session=%s failed (%T)", session.ID, err) + continue + } + if running { + h.observeUnattended(session.ID, broker) + } + } +} + +// observeUnattended opens (or reuses) the session's room with no client and +// observes its stream until the turn the harness reported has ended. +func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) { + h.mu.Lock() + if h.rooms[sessionID] == nil && h.unattendedCountLocked(nil) >= chatUnattendedLimit { + h.mu.Unlock() + log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit) + return + } + room := h.roomLocked(sessionID) + if room.broker == nil { + room.broker = broker + } + if len(room.subs) == 0 { + room.holdUntil = h.clock().Add(h.unattended.settle) + h.watchUnattendedLocked(room) + } + h.mu.Unlock() + if err := h.ensureObserving(room, broker); err != nil { + if room.ctx.Err() == nil { + log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err) + } + return + } + h.startRoomPoller(room, broker) +} diff --git a/internal/server/chat_unattended_test.go b/internal/server/chat_unattended_test.go new file mode 100644 index 00000000..9d0d0b6d --- /dev/null +++ b/internal/server/chat_unattended_test.go @@ -0,0 +1,346 @@ +package server + +import ( + "context" + "path/filepath" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// observedChat is a channelChat whose stream can be opened more than once and +// reports how many streams are live, so a test can prove when the hub observes +// a session with no client attached and when it stops. probe answers the +// server-start ProbeTurn. +type observedChat struct { + *channelChat + smu sync.Mutex + active int + opened int + probe map[string]bool + probes int +} + +func newObservedChat() *observedChat { + return &observedChat{channelChat: newChannelChat(), probe: map[string]bool{}} +} + +func (c *observedChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error { + c.smu.Lock() + c.active++ + c.opened++ + c.smu.Unlock() + defer func() { + c.smu.Lock() + c.active-- + c.smu.Unlock() + }() + if ready != nil { + ready() + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case frame := <-c.frames: + if err := emit(frame); err != nil { + return err + } + } + } +} + +func (c *observedChat) ProbeTurn(_ context.Context, sessionID string) (bool, error) { + c.smu.Lock() + defer c.smu.Unlock() + c.probes++ + return c.probe[sessionID], nil +} + +func (c *observedChat) streams() (active, opened int) { + c.smu.Lock() + defer c.smu.Unlock() + return c.active, c.opened +} + +func fastUnattended(t *testing.T) { + t.Helper() + check, settle := chatUnattendedCheck, chatUnattendedRecoverSettle + chatUnattendedCheck = 10 * time.Millisecond + chatUnattendedRecoverSettle = 200 * time.Millisecond + t.Cleanup(func() { chatUnattendedCheck, chatUnattendedRecoverSettle = check, settle }) +} + +func unattendedStore(t *testing.T, replies bool) *pipeline.Store { + t.Helper() + store := openQueueStore(t, filepath.Join(t.TempDir(), "chat.db")) + if err := store.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + if replies { + if _, err := store.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil { + t.Fatal(err) + } + } + return store +} + +// chatNotes returns the recorded app notifications of one kind. +func chatNotes(t *testing.T, store *pipeline.Store, kind string) []protocol.AppNotification { + t.Helper() + active, _, err := store.ActiveAppNotifications(context.Background(), "", time.Now()) + if err != nil { + t.Fatal(err) + } + var out []protocol.AppNotification + for _, n := range active { + if n.Kind == kind { + out = append(out, n) + } + } + return out +} + +func stepFrameFor(status, turn string) protocol.ChatEventFrame { + return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: status, TurnID: turn}} +} + +func textFrame(text string) protocol.ChatEventFrame { + return protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventText, MessageID: "msg_a", Part: &protocol.ChatPart{ID: "msg_a_p"}, Delta: text} +} + +// A turn started from the app keeps being observed after the app closes, so +// its finished reply is recorded (app stream) and published (ntfy) once, and +// the observer is torn down when the turn ends. +func TestChatReplyNotifiesAfterLastClientLeaves(t *testing.T) { + fastUnattended(t) + captured := newNtfyCapture(t) + store := unattendedStore(t, true) + broker := newObservedChat() + server := startQueueServer(t, broker, store) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1") + waitFor(t, "prompt", func() bool { return broker.promptCount() == 1 }) + broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) + broker.push(textFrame("The build is green.")) + conn.CloseNow() + time.Sleep(100 * time.Millisecond) // let the server notice the disconnect + if active, _ := broker.streams(); active != 1 { + t.Fatalf("in-flight turn not observed after the client left: %d streams", active) + } + broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1")) + waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 }) + waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 }) + waitFor(t, "ntfy publish", func() bool { return len(captured()) == 1 }) + time.Sleep(50 * time.Millisecond) + notes := chatNotes(t, store, protocol.NotificationChatReply) + if got := captured(); len(notes) != 1 || len(got) != 1 || got[0].Title != "Release prep replied" || got[0].Message != "The build is green." { + t.Fatalf("notes=%+v ntfy=%+v", notes, got) + } + if notes[0].Message != "The build is green." || notes[0].Click != "pipelines://chat/ses_1" { + t.Fatalf("note = %+v", notes[0]) + } +} + +// The chat_reply_finished setting still decides: off (the default) records +// nothing, even though the unattended turn is observed. +func TestChatUnattendedReplyRespectsSetting(t *testing.T) { + fastUnattended(t) + t.Setenv("PIPELINES_NTFY_URL", "") + store := unattendedStore(t, false) + broker := newObservedChat() + server := startQueueServer(t, broker, store) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1") + broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) + conn.CloseNow() + time.Sleep(100 * time.Millisecond) + broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1")) + waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 }) + time.Sleep(100 * time.Millisecond) + if notes := chatNotes(t, store, protocol.NotificationChatReply); len(notes) != 0 { + t.Fatalf("reply recorded with chat_reply_finished off: %+v", notes) + } +} + +// A question the agent asks while no client is attached notifies. +func TestChatQuestionNotifiesWithNoClient(t *testing.T) { + fastUnattended(t) + t.Setenv("PIPELINES_NTFY_URL", "") + store := unattendedStore(t, false) + broker := newObservedChat() + server := startQueueServer(t, broker, store) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "ship?", "op-1") + broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) + conn.CloseNow() + time.Sleep(100 * time.Millisecond) + form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Which branch should ship?"} + broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form}) + waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 }) + if notes := chatNotes(t, store, protocol.NotificationChatQuestion); notes[0].Message != "Which branch should ship?" { + t.Fatalf("note = %+v", notes[0]) + } + // The turn ends with the question withdrawn: nothing is left to observe. + form.Cancelled = true + broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form}) + broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1")) + waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 }) +} + +// A client reattaching mid-turn joins the observing room: the reply is still +// announced exactly once and the client sees the turn end. +func TestChatReattachMidTurnDoesNotDuplicate(t *testing.T) { + fastUnattended(t) + captured := newNtfyCapture(t) + store := unattendedStore(t, true) + broker := newObservedChat() + server := startQueueServer(t, broker, store) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1") + broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) + broker.push(textFrame("Done.")) + form := protocol.ChatForm{ID: "que_1", SessionID: "ses_1", Kind: "question", Title: "Continue?"} + broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form}) + waitFor(t, "question notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatQuestion)) == 1 }) + conn.CloseNow() + time.Sleep(100 * time.Millisecond) + + again := dialChatConn(t, server.ctx, server.srv) + writeChat(t, server.ctx, again, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}) + readUntil(t, server.ctx, again, protocol.ChatSubscribe) + if _, opened := broker.streams(); opened != 1 { + t.Fatalf("reattach opened a second stream: %d", opened) + } + form.Resolved = true + broker.push(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventForm, Form: &form}) + broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1")) + for { + msg := readUntil(t, server.ctx, again, protocol.ChatEvent) + if e := msg.ChatEvent; e.Kind == protocol.ChatEventStep && e.Step != nil && e.Step.Status == protocol.ChatStepEnded { + break + } + } + waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 }) + time.Sleep(100 * time.Millisecond) + if got := captured(); len(got) != 2 { + t.Fatalf("ntfy deliveries = %+v, want one question and one reply", got) + } + if n := len(chatNotes(t, store, protocol.NotificationChatQuestion)) + len(chatNotes(t, store, protocol.NotificationChatReply)); n > 2 { + t.Fatalf("duplicate notifications: %d", n) + } + // The attended room is idle now; the client leaving releases it. + again.CloseNow() + waitFor(t, "idle room released", func() bool { active, _ := broker.streams(); return active == 0 }) +} + +// An idle session is not observed once its last client leaves, and a server +// start does not observe sessions whose harness reports no running turn. +func TestChatIdleSessionWithNoClientIsNotObserved(t *testing.T) { + fastUnattended(t) + t.Setenv("PIPELINES_NTFY_URL", "") + store := unattendedStore(t, true) + broker := newObservedChat() + server := startQueueServer(t, broker, store) + waitFor(t, "start-up probe", func() bool { + broker.smu.Lock() + defer broker.smu.Unlock() + return broker.probes == 1 + }) + if _, opened := broker.streams(); opened != 0 { + t.Fatalf("idle session observed at start: %d streams", opened) + } + conn := dialChatConn(t, server.ctx, server.srv) + writeChat(t, server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}) + readUntil(t, server.ctx, conn, protocol.ChatSubscribe) + waitFor(t, "stream", func() bool { active, _ := broker.streams(); return active == 1 }) + conn.CloseNow() + waitFor(t, "idle stream released", func() bool { active, _ := broker.streams(); return active == 0 }) +} + +// A turn a previous server process left running is observed again at start, +// with no client, so its reply still notifies; then the observer goes away. +func TestChatRunningTurnObservedAfterRestart(t *testing.T) { + fastUnattended(t) + t.Setenv("PIPELINES_NTFY_URL", "") + store := unattendedStore(t, true) + broker := newObservedChat() + broker.probe["ses_1"] = true + startQueueServer(t, broker, store) + waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 }) + // The harness re-announces the running turn on attach, then finishes it. + broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_9")) + broker.push(textFrame("Finished after the restart.")) + broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_9")) + waitFor(t, "reply notification", func() bool { return len(chatNotes(t, store, protocol.NotificationChatReply)) == 1 }) + waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 }) +} + +// A recovered session whose stream never reports the turn is released after +// the settle window rather than observed forever. +func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) { + fastUnattended(t) + t.Setenv("PIPELINES_NTFY_URL", "") + store := unattendedStore(t, true) + broker := newObservedChat() + broker.probe["ses_1"] = true + startQueueServer(t, broker, store) + waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 }) + waitFor(t, "released after settle", func() bool { active, _ := broker.streams(); return active == 0 }) +} + +// A turn that never reports its end does not hold an observer forever. +func TestChatStuckUnattendedTurnIsReleased(t *testing.T) { + fastUnattended(t) + t.Setenv("PIPELINES_NTFY_URL", "") + limit := chatUnattendedTurnLimit + chatUnattendedTurnLimit = 300 * time.Millisecond + t.Cleanup(func() { chatUnattendedTurnLimit = limit }) + store := unattendedStore(t, true) + broker := newObservedChat() + server := startQueueServer(t, broker, store) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1") + broker.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) + conn.CloseNow() + time.Sleep(100 * time.Millisecond) + if active, _ := broker.streams(); active != 1 { + t.Fatalf("in-flight turn not observed: %d", active) + } + waitFor(t, "stuck observer released", func() bool { active, _ := broker.streams(); return active == 0 }) +} + +// At most chatUnattendedLimit chats are kept observing with no client. +func TestChatUnattendedRoomsAreBounded(t *testing.T) { + fastUnattended(t) + h := newChatHub() + defer func() { + h.mu.Lock() + for _, room := range h.rooms { + room.cancel() + } + h.mu.Unlock() + }() + kept := 0 + for i := 0; i < chatUnattendedLimit+3; i++ { + id := "ses_" + string(rune('a'+i)) + h.mu.Lock() + room := h.roomLocked(id) + room.running = true + sub := &chatSub{hub: h, room: room, clientID: "c", ch: make(chan protocol.Message, 1), done: make(chan struct{})} + room.subs["c"] = sub + h.mu.Unlock() + h.detach(id, sub) + h.mu.Lock() + if h.rooms[id] == room { + kept++ + } + h.mu.Unlock() + } + if kept != chatUnattendedLimit { + t.Fatalf("kept %d unattended rooms, want %d", kept, chatUnattendedLimit) + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 4d368653..abd42103 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -115,6 +115,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin settingsStore = stores[0] } hub.settings = settingsStore + if options.Chat != nil && registry != nil { + // Keep observing chats whose turn a previous process left running, so + // their reply or question still notifies with no client attached. + go hub.observeRunningTurns(ctx, options.Chat) + } if len(stores) > 0 { // Prompts queued behind a running turn are durable and owned by their // session; resume delivering any a previous process left queued.