Files
pipelines/internal/server/chat_unattended.go
T
alan 1b98877d88 Chat list activity: turn running and background tasks on every row, pushed to every connection
chat_sessions rows carry activity (turn_running, background_tasks, seq) from
the hub's own state, and chat_activity is pushed to every connection when a
session's turn starts or ends or its running background-task count changes.
A room whose last viewer left stays observed while background tasks run,
bounded by the unattended limits; a Claude Code process exit ends its tasks.
2026-11-03 19:45:43 +00:00

270 lines
9.4 KiB
Go

package server
import (
"context"
"log"
"time"
)
// Chat attention notifications (chat_question, chat_approval, chat_reply) are
// derived from a session's live event stream, and the hub only reads that
// stream while the session has a room. A room normally lives while a client
// watches the chat, but the operator relies on these notifications precisely
// when no app is open. So a room whose last client leaves stays observing,
// unattended, while its turn is in flight, it holds an unanswered question or
// approval, or background tasks it started are still running (so the chat list
// sees them finish), and is torn down once that work is done. The same unattended
// observation is started at server start for sessions whose harness reports a
// running turn, so a reply that finishes after a restart still notifies.
//
// Resources are bounded: at most chatUnattendedLimit rooms are kept without a
// client (rooms holding queued prompts are not counted: they were always kept
// until delivered), a stuck turn or a background task that never reports an
// exit is released after chatUnattendedTurnLimit, and an idle session is never
// observed.
// chatUnattendedLimit bounds the rooms kept observing with no client attached
// and no queued prompt.
const chatUnattendedLimit = 16
// chatRecoverSessionLimit bounds how many recently used open sessions are
// probed for a running turn at server start.
const chatRecoverSessionLimit = 32
var (
// chatUnattendedCheck is how often an unattended room re-checks whether it
// still has a reason to observe its session.
chatUnattendedCheck = 5 * time.Second
// chatUnattendedTurnLimit releases an unattended room whose turn never
// reports an end (a stuck harness or a lost end event).
chatUnattendedTurnLimit = 6 * time.Hour
// chatUnattendedRecoverSettle keeps a room recovered at start alive until
// its freshly attached stream has had time to announce the running turn.
chatUnattendedRecoverSettle = 15 * time.Second
// chatRecoverWindow limits start-up probing to sessions used recently.
chatRecoverWindow = 24 * time.Hour
)
// chatUnattendedTimings is the hub's copy of the timings above, taken when the
// hub is built so its goroutines never read the package variables.
type chatUnattendedTimings struct {
check, turnLimit, settle, window time.Duration
}
func currentChatUnattendedTimings() chatUnattendedTimings {
return chatUnattendedTimings{check: chatUnattendedCheck, turnLimit: chatUnattendedTurnLimit, settle: chatUnattendedRecoverSettle, window: chatRecoverWindow}
}
// chatTurnProber is implemented by a broker that can ask the harness itself,
// with I/O, whether a session's turn is in flight. It is used once at server
// start, when the in-memory turn state is empty.
type chatTurnProber interface {
ProbeTurn(ctx context.Context, sessionID string) (bool, error)
}
// keepUnattendedLocked reports whether a room with no client still has work
// the operator must hear about: a turn in flight (the runner's, or one its
// stream announced), an unresolved approval or question, or a harness that
// reports its own turn running (busy, read by the caller off the hub mutex),
// or background tasks still running.
// hold honours the start-up settle window. Queued prompts are the caller's
// concern. Callers hold h.mu.
func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool {
if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 {
return true
}
// Running background tasks keep the stream observed so their exits are
// seen and the chat list's indicator clears. This is bounded like a turn:
// the unattended room limit, the chatUnattendedTurnLimit expiry, and the
// harness itself (a Claude Code process that exits ends its tasks).
if h.backgroundTasksLocked(room.sessionID) > 0 {
return true
}
return hold && h.clock().Before(room.holdUntil)
}
// unattendedCountLocked counts the rooms kept observing with no client and no
// queued prompt, other than except. Callers hold h.mu.
func (h *chatHub) unattendedCountLocked(except *chatRoom) int {
n := 0
for _, room := range h.rooms {
if room != except && room.watching && len(room.subs) == 0 && len(room.queue) == 0 {
n++
}
}
return n
}
// leaveUnattendedLocked decides the fate of a room whose last client just
// left: a room holding queued prompts stays for its runner (as before); a room
// with an in-flight turn or pending attention keeps observing unattended,
// within the bound; anything else is released. Callers hold h.mu.
func (h *chatHub) leaveUnattendedLocked(room *chatRoom) {
if len(room.queue) > 0 {
h.watchUnattendedLocked(room)
return
}
if !h.keepUnattendedLocked(room, false, true) {
h.releaseRoomLocked(room)
return
}
if !room.watching && h.unattendedCountLocked(room) >= chatUnattendedLimit {
log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit)
h.releaseRoomLocked(room)
return
}
h.watchUnattendedLocked(room)
}
// watchUnattendedLocked starts the room's unattended watcher once. Callers
// hold h.mu.
func (h *chatHub) watchUnattendedLocked(room *chatRoom) {
if room.watching || room.ctx.Err() != nil {
return
}
room.watching = true
room.unattendedSince = h.clock()
go h.watchUnattended(room)
}
func (h *chatHub) watchUnattended(room *chatRoom) {
check := h.unattended.check
if check <= 0 {
check = 5 * time.Second
}
ticker := time.NewTicker(check)
defer ticker.Stop()
for {
select {
case <-room.ctx.Done():
return
case <-ticker.C:
}
if h.reapUnattended(room) {
return
}
}
}
// reapUnattended re-checks one unattended room and reports whether its
// watcher is finished: a client reattached (the room is attended again), or
// the room had nothing left to observe, or its turn exceeded the limit, and
// was released.
func (h *chatHub) reapUnattended(room *chatRoom) bool {
h.mu.Lock()
if done := h.attendedLocked(room); done {
h.mu.Unlock()
return true
}
broker := room.broker
h.mu.Unlock()
// TurnRunning performs no I/O but may take the broker's own lock, so it is
// never asked under the hub mutex.
busy := brokerTurnRunning(broker, room.sessionID)
h.mu.Lock()
defer h.mu.Unlock()
if done := h.attendedLocked(room); done {
return true
}
if len(room.queue) > 0 {
// The runner owns a room with queued prompts until they are delivered;
// the stuck-turn limit counts from the last of them.
room.unattendedSince = h.clock()
return false
}
expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit
if !expired && h.keepUnattendedLocked(room, busy, true) {
return false
}
if expired {
log.Printf("chat session %s: released the unattended observer after %s without its turn or background tasks ending", room.sessionID, h.unattended.turnLimit)
}
room.watching = false
h.releaseRoomLocked(room)
return true
}
// attendedLocked reports whether the watcher should stop because the room is
// gone or a client is watching it again. Callers hold h.mu.
func (h *chatHub) attendedLocked(room *chatRoom) bool {
if h.rooms[room.sessionID] != room || room.ctx.Err() != nil {
room.watching = false
return true
}
if len(room.subs) > 0 {
room.watching = false
room.unattendedSince = time.Time{}
return true
}
return false
}
// observeRunningTurns is the start-up half: each recently used open operator
// session whose harness reports a turn in flight gets an unattended room
// observing its stream, so the reply (or a question) still notifies. It is
// best effort per harness: OpenCode and Codex keep running across a Pipelines
// restart and report it; Claude Code runs as a child of the server and so has
// no turn left after a restart.
func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
prober, ok := broker.(chatTurnProber)
if !ok || h.registry == nil {
return
}
sessions, err := h.registry.OperatorSessions(ctx)
if err != nil {
log.Printf("chat turn recovery: %v", err)
return
}
cutoff := h.clock().Add(-h.unattended.window).UnixMilli()
probed := 0
for _, session := range sessions {
if ctx.Err() != nil {
return
}
if session.ClosedAt != 0 || session.UpdatedAt < cutoff || !validChatSessionID(session.ID) {
continue
}
if probed >= chatRecoverSessionLimit {
break
}
probed++
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
running, err := prober.ProbeTurn(probeCtx, session.ID)
cancel()
if err != nil {
log.Printf("chat turn recovery probe session=%s failed (%T)", session.ID, err)
continue
}
if running {
h.observeUnattended(session.ID, broker)
}
}
}
// observeUnattended opens (or reuses) the session's room with no client and
// observes its stream until the turn the harness reported has ended.
func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
h.mu.Lock()
if h.rooms[sessionID] == nil && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
h.mu.Unlock()
log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit)
return
}
room := h.roomLocked(sessionID)
if room.broker == nil {
room.broker = broker
}
if len(room.subs) == 0 {
room.holdUntil = h.clock().Add(h.unattended.settle)
h.watchUnattendedLocked(room)
}
h.mu.Unlock()
if err := h.ensureObserving(room, broker); err != nil {
if room.ctx.Err() == nil {
log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err)
}
return
}
h.startRoomPoller(room, broker)
}