Files

285 lines
9.7 KiB
Go

package server
import (
"context"
"log"
"sync"
"time"
)
// Chat attention notifications (chat_question, chat_approval, chat_reply) are
// derived from a session's live event stream, and the hub only reads that
// stream while the session has a room. A room normally lives while a client
// watches the chat, but the operator relies on these notifications precisely
// when no app is open. So a room whose last client leaves stays observing,
// unattended, while its turn is in flight, it holds an unanswered question or
// approval, or background tasks it started are still running (so the chat list
// sees them finish), and is torn down once that work is done. The same unattended
// observation is started at server start for sessions whose harness reports a
// running turn, so a reply that finishes after a restart still notifies.
//
// Idle/stale foreground observers are bounded by chatUnattendedLimit and
// chatUnattendedTurnLimit. Known running background work (like queued prompts)
// is not an idle lease: it remains attached until actual completion or explicit
// stop. Its durable journal is independent of bounded replay/room retention.
// chatUnattendedLimit bounds the rooms kept observing with no client attached
// and no queued prompt.
const chatUnattendedLimit = 16
// chatRecoverSessionLimit bounds how many recently used open sessions are
// probed for a running turn at server start.
const chatRecoverSessionLimit = 32
var (
// chatUnattendedCheck is how often an unattended room re-checks whether it
// still has a reason to observe its session.
chatUnattendedCheck = 5 * time.Second
// chatUnattendedTurnLimit releases an unattended room whose turn never
// reports an end (a stuck harness or a lost end event).
chatUnattendedTurnLimit = 6 * time.Hour
// chatUnattendedRecoverSettle keeps a room recovered at start alive until
// its freshly attached stream has had time to announce the running turn.
chatUnattendedRecoverSettle = 15 * time.Second
// chatRecoverWindow limits start-up probing to sessions used recently.
chatRecoverWindow = 24 * time.Hour
)
// chatUnattendedTimings is the hub's copy of the timings above, taken when the
// hub is built so its goroutines never read the package variables.
type chatUnattendedTimings struct {
check, turnLimit, settle, window time.Duration
}
func currentChatUnattendedTimings() chatUnattendedTimings {
return chatUnattendedTimings{check: chatUnattendedCheck, turnLimit: chatUnattendedTurnLimit, settle: chatUnattendedRecoverSettle, window: chatRecoverWindow}
}
// chatTurnProber is implemented by a broker that can ask the harness itself,
// with I/O, whether a session's turn is in flight. It is used once at server
// start, when the in-memory turn state is empty.
type chatTurnProber interface {
ProbeTurn(ctx context.Context, sessionID string) (bool, error)
}
// keepUnattendedLocked reports whether a room with no client still has work
// the operator must hear about: a turn in flight (the runner's, or one its
// stream announced), an unresolved approval or question, or a harness that
// reports its own turn running (busy, read by the caller off the hub mutex),
// or background tasks still running.
// hold honours the start-up settle window. Queued prompts are the caller's
// concern. Callers hold h.mu.
func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool {
if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 {
return true
}
// Running background tasks keep the stream observed so their actual exits
// are seen and the chat list's indicator clears, without a fixed age cap.
if h.backgroundTasksLocked(room.sessionID) > 0 {
return true
}
return hold && h.clock().Before(room.holdUntil)
}
// unattendedCountLocked counts the rooms kept observing with no client and no
// queued prompt, other than except. Callers hold h.mu.
func (h *chatHub) unattendedCountLocked(except *chatRoom) int {
n := 0
for _, room := range h.rooms {
if room != except && room.watching && len(room.subs) == 0 && len(room.queue) == 0 {
n++
}
}
return n
}
// leaveUnattendedLocked decides the fate of a room whose last client just
// left: a room holding queued prompts stays for its runner (as before); a room
// with an in-flight turn or pending attention keeps observing unattended,
// within the bound; anything else is released. Callers hold h.mu.
func (h *chatHub) leaveUnattendedLocked(room *chatRoom) {
if len(room.queue) > 0 {
h.watchUnattendedLocked(room)
return
}
if !h.keepUnattendedLocked(room, false, true) {
h.releaseRoomLocked(room)
return
}
if !room.watching && h.backgroundTasksLocked(room.sessionID) == 0 && h.unattendedCountLocked(room) >= chatUnattendedLimit {
log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit)
h.releaseRoomLocked(room)
return
}
h.watchUnattendedLocked(room)
}
// watchUnattendedLocked starts the room's unattended watcher once. Callers
// hold h.mu.
func (h *chatHub) watchUnattendedLocked(room *chatRoom) {
if room.watching || room.ctx.Err() != nil {
return
}
room.watching = true
room.unattendedSince = h.clock()
go h.watchUnattended(room)
}
func (h *chatHub) watchUnattended(room *chatRoom) {
check := h.unattended.check
if check <= 0 {
check = 5 * time.Second
}
ticker := time.NewTicker(check)
defer ticker.Stop()
for {
select {
case <-room.ctx.Done():
return
case <-ticker.C:
}
if h.reapUnattended(room) {
return
}
}
}
// reapUnattended re-checks one unattended room and reports whether its
// watcher is finished: a client reattached (the room is attended again), or
// the room had nothing left to observe, or its turn exceeded the limit, and
// was released.
func (h *chatHub) reapUnattended(room *chatRoom) bool {
h.mu.Lock()
if done := h.attendedLocked(room); done {
h.mu.Unlock()
return true
}
broker := room.broker
h.mu.Unlock()
// TurnRunning performs no I/O but may take the broker's own lock, so it is
// never asked under the hub mutex.
busy := brokerTurnRunning(broker, room.sessionID)
h.mu.Lock()
defer h.mu.Unlock()
if done := h.attendedLocked(room); done {
return true
}
if len(room.queue) > 0 {
// The runner owns a room with queued prompts until they are delivered;
// the stuck-turn limit counts from the last of them.
room.unattendedSince = h.clock()
return false
}
// A running task is actual work, not an idle-observer lease. Its terminal
// lifecycle/readback, or explicit operator stop, releases it normally.
if h.backgroundTasksLocked(room.sessionID) > 0 {
return false
}
expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit
if !expired && h.keepUnattendedLocked(room, busy, true) {
return false
}
if expired {
log.Printf("chat session %s: released the unattended observer after %s without its turn or background tasks ending", room.sessionID, h.unattended.turnLimit)
}
room.watching = false
h.releaseRoomLocked(room)
return true
}
// attendedLocked reports whether the watcher should stop because the room is
// gone or a client is watching it again. Callers hold h.mu.
func (h *chatHub) attendedLocked(room *chatRoom) bool {
if h.rooms[room.sessionID] != room || room.ctx.Err() != nil {
room.watching = false
return true
}
if len(room.subs) > 0 {
room.watching = false
room.unattendedSince = time.Time{}
return true
}
return false
}
// observeRunningTurns discovers live foreground or background work even when
// no observer survived long enough to journal it (including a first upgrade or
// an abrupt old-server exit). Every open registered session is considered; old
// metadata is not evidence that a long-running task ended. Work is bounded to
// batches of 32 with four concurrent readback probes, never prompt/create/resume.
func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
if h.registry == nil {
return
}
sessions, err := h.registry.OperatorSessions(ctx)
if err != nil {
log.Printf("chat recovery registry: %v", err)
return
}
ids := make([]string, 0, len(sessions))
for _, session := range sessions {
if session.ClosedAt == 0 && validChatSessionID(session.ID) {
ids = append(ids, session.ID)
}
}
for start := 0; start < len(ids); start += chatRecoverSessionLimit {
if ctx.Err() != nil {
return
}
end := start + chatRecoverSessionLimit
if end > len(ids) {
end = len(ids)
}
jobs := make(chan string)
var workers sync.WaitGroup
for i := 0; i < 4; i++ {
workers.Add(1)
go func() {
defer workers.Done()
for id := range jobs {
h.recoverDiscoveredSession(ctx, broker, id)
}
}()
}
for _, id := range ids[start:end] {
select {
case jobs <- id:
case <-ctx.Done():
close(jobs)
workers.Wait()
return
}
}
close(jobs)
workers.Wait()
}
}
// observeUnattended opens (or reuses) the session's room with no client and
// observes its stream until the turn the harness reported has ended.
func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
h.mu.Lock()
if h.rooms[sessionID] == nil && h.backgroundTasksLocked(sessionID) == 0 && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
h.mu.Unlock()
log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit)
return
}
room := h.roomLocked(sessionID)
if room.broker == nil {
room.broker = broker
}
if len(room.subs) == 0 {
room.holdUntil = h.clock().Add(h.unattended.settle)
h.watchUnattendedLocked(room)
}
h.mu.Unlock()
if err := h.ensureObserving(room, broker); err != nil {
if room.ctx.Err() == nil {
log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err)
go h.retryRoomObservation(room, broker)
}
return
}
h.startRoomPoller(room, broker)
}