package server import ( "context" "log" "sync" "time" ) // Chat attention notifications (chat_question, chat_approval, chat_reply) are // derived from a session's live event stream, and the hub only reads that // stream while the session has a room. A room normally lives while a client // watches the chat, but the operator relies on these notifications precisely // when no app is open. So a room whose last client leaves stays observing, // unattended, while its turn is in flight, it holds an unanswered question or // approval, or background tasks it started are still running (so the chat list // sees them finish), and is torn down once that work is done. The same unattended // observation is started at server start for sessions whose harness reports a // running turn, so a reply that finishes after a restart still notifies. // // Idle/stale foreground observers are bounded by chatUnattendedLimit and // chatUnattendedTurnLimit. Known running background work (like queued prompts) // is not an idle lease: it remains attached until actual completion or explicit // stop. Its durable journal is independent of bounded replay/room retention. // chatUnattendedLimit bounds the rooms kept observing with no client attached // and no queued prompt. const chatUnattendedLimit = 16 // chatRecoverSessionLimit bounds how many recently used open sessions are // probed for a running turn at server start. const chatRecoverSessionLimit = 32 var ( // chatUnattendedCheck is how often an unattended room re-checks whether it // still has a reason to observe its session. chatUnattendedCheck = 5 * time.Second // chatUnattendedTurnLimit releases an unattended room whose turn never // reports an end (a stuck harness or a lost end event). chatUnattendedTurnLimit = 6 * time.Hour // chatUnattendedRecoverSettle keeps a room recovered at start alive until // its freshly attached stream has had time to announce the running turn. chatUnattendedRecoverSettle = 15 * time.Second // chatRecoverWindow limits start-up probing to sessions used recently. chatRecoverWindow = 24 * time.Hour ) // chatUnattendedTimings is the hub's copy of the timings above, taken when the // hub is built so its goroutines never read the package variables. type chatUnattendedTimings struct { check, turnLimit, settle, window time.Duration } func currentChatUnattendedTimings() chatUnattendedTimings { return chatUnattendedTimings{check: chatUnattendedCheck, turnLimit: chatUnattendedTurnLimit, settle: chatUnattendedRecoverSettle, window: chatRecoverWindow} } // chatTurnProber is implemented by a broker that can ask the harness itself, // with I/O, whether a session's turn is in flight. It is used once at server // start, when the in-memory turn state is empty. type chatTurnProber interface { ProbeTurn(ctx context.Context, sessionID string) (bool, error) } // keepUnattendedLocked reports whether a room with no client still has work // the operator must hear about: a turn in flight (the runner's, or one its // stream announced), an unresolved approval or question, or a harness that // reports its own turn running (busy, read by the caller off the hub mutex), // or background tasks still running. // hold honours the start-up settle window. Queued prompts are the caller's // concern. Callers hold h.mu. func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool { if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 { return true } // Running background tasks keep the stream observed so their actual exits // are seen and the chat list's indicator clears, without a fixed age cap. if h.backgroundTasksLocked(room.sessionID) > 0 { return true } return hold && h.clock().Before(room.holdUntil) } // unattendedCountLocked counts the rooms kept observing with no client and no // queued prompt, other than except. Callers hold h.mu. func (h *chatHub) unattendedCountLocked(except *chatRoom) int { n := 0 for _, room := range h.rooms { if room != except && room.watching && len(room.subs) == 0 && len(room.queue) == 0 { n++ } } return n } // leaveUnattendedLocked decides the fate of a room whose last client just // left: a room holding queued prompts stays for its runner (as before); a room // with an in-flight turn or pending attention keeps observing unattended, // within the bound; anything else is released. Callers hold h.mu. func (h *chatHub) leaveUnattendedLocked(room *chatRoom) { if len(room.queue) > 0 { h.watchUnattendedLocked(room) return } if !h.keepUnattendedLocked(room, false, true) { h.releaseRoomLocked(room) return } if !room.watching && h.backgroundTasksLocked(room.sessionID) == 0 && h.unattendedCountLocked(room) >= chatUnattendedLimit { log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit) h.releaseRoomLocked(room) return } h.watchUnattendedLocked(room) } // watchUnattendedLocked starts the room's unattended watcher once. Callers // hold h.mu. func (h *chatHub) watchUnattendedLocked(room *chatRoom) { if room.watching || room.ctx.Err() != nil { return } room.watching = true room.unattendedSince = h.clock() go h.watchUnattended(room) } func (h *chatHub) watchUnattended(room *chatRoom) { check := h.unattended.check if check <= 0 { check = 5 * time.Second } ticker := time.NewTicker(check) defer ticker.Stop() for { select { case <-room.ctx.Done(): return case <-ticker.C: } if h.reapUnattended(room) { return } } } // reapUnattended re-checks one unattended room and reports whether its // watcher is finished: a client reattached (the room is attended again), or // the room had nothing left to observe, or its turn exceeded the limit, and // was released. func (h *chatHub) reapUnattended(room *chatRoom) bool { h.mu.Lock() if done := h.attendedLocked(room); done { h.mu.Unlock() return true } broker := room.broker h.mu.Unlock() // TurnRunning performs no I/O but may take the broker's own lock, so it is // never asked under the hub mutex. busy := brokerTurnRunning(broker, room.sessionID) h.mu.Lock() defer h.mu.Unlock() if done := h.attendedLocked(room); done { return true } if len(room.queue) > 0 { // The runner owns a room with queued prompts until they are delivered; // the stuck-turn limit counts from the last of them. room.unattendedSince = h.clock() return false } // A running task is actual work, not an idle-observer lease. Its terminal // lifecycle/readback, or explicit operator stop, releases it normally. if h.backgroundTasksLocked(room.sessionID) > 0 { return false } expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit if !expired && h.keepUnattendedLocked(room, busy, true) { return false } if expired { log.Printf("chat session %s: released the unattended observer after %s without its turn or background tasks ending", room.sessionID, h.unattended.turnLimit) } room.watching = false h.releaseRoomLocked(room) return true } // attendedLocked reports whether the watcher should stop because the room is // gone or a client is watching it again. Callers hold h.mu. func (h *chatHub) attendedLocked(room *chatRoom) bool { if h.rooms[room.sessionID] != room || room.ctx.Err() != nil { room.watching = false return true } if len(room.subs) > 0 { room.watching = false room.unattendedSince = time.Time{} return true } return false } // observeRunningTurns discovers live foreground or background work even when // no observer survived long enough to journal it (including a first upgrade or // an abrupt old-server exit). Every open registered session is considered; old // metadata is not evidence that a long-running task ended. Work is bounded to // batches of 32 with four concurrent readback probes, never prompt/create/resume. func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) { if h.registry == nil { return } sessions, err := h.registry.OperatorSessions(ctx) if err != nil { log.Printf("chat recovery registry: %v", err) return } ids := make([]string, 0, len(sessions)) for _, session := range sessions { if session.ClosedAt == 0 && validChatSessionID(session.ID) { ids = append(ids, session.ID) } } for start := 0; start < len(ids); start += chatRecoverSessionLimit { if ctx.Err() != nil { return } end := start + chatRecoverSessionLimit if end > len(ids) { end = len(ids) } jobs := make(chan string) var workers sync.WaitGroup for i := 0; i < 4; i++ { workers.Add(1) go func() { defer workers.Done() for id := range jobs { h.recoverDiscoveredSession(ctx, broker, id) } }() } for _, id := range ids[start:end] { select { case jobs <- id: case <-ctx.Done(): close(jobs) workers.Wait() return } } close(jobs) workers.Wait() } } // observeUnattended opens (or reuses) the session's room with no client and // observes its stream until the turn the harness reported has ended. func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) { h.mu.Lock() if h.rooms[sessionID] == nil && h.backgroundTasksLocked(sessionID) == 0 && h.unattendedCountLocked(nil) >= chatUnattendedLimit { h.mu.Unlock() log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit) return } room := h.roomLocked(sessionID) if room.broker == nil { room.broker = broker } if len(room.subs) == 0 { room.holdUntil = h.clock().Add(h.unattended.settle) h.watchUnattendedLocked(room) } h.mu.Unlock() if err := h.ensureObserving(room, broker); err != nil { if room.ctx.Err() == nil { log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err) go h.retryRoomObservation(room, broker) } return } h.startRoomPoller(room, broker) }