285 lines
9.7 KiB
Go
285 lines
9.7 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// Chat attention notifications (chat_question, chat_approval, chat_reply) are
|
|
// derived from a session's live event stream, and the hub only reads that
|
|
// stream while the session has a room. A room normally lives while a client
|
|
// watches the chat, but the operator relies on these notifications precisely
|
|
// when no app is open. So a room whose last client leaves stays observing,
|
|
// unattended, while its turn is in flight, it holds an unanswered question or
|
|
// approval, or background tasks it started are still running (so the chat list
|
|
// sees them finish), and is torn down once that work is done. The same unattended
|
|
// observation is started at server start for sessions whose harness reports a
|
|
// running turn, so a reply that finishes after a restart still notifies.
|
|
//
|
|
// Idle/stale foreground observers are bounded by chatUnattendedLimit and
|
|
// chatUnattendedTurnLimit. Known running background work (like queued prompts)
|
|
// is not an idle lease: it remains attached until actual completion or explicit
|
|
// stop. Its durable journal is independent of bounded replay/room retention.
|
|
|
|
// chatUnattendedLimit bounds the rooms kept observing with no client attached
|
|
// and no queued prompt.
|
|
const chatUnattendedLimit = 16
|
|
|
|
// chatRecoverSessionLimit bounds how many recently used open sessions are
|
|
// probed for a running turn at server start.
|
|
const chatRecoverSessionLimit = 32
|
|
|
|
var (
|
|
// chatUnattendedCheck is how often an unattended room re-checks whether it
|
|
// still has a reason to observe its session.
|
|
chatUnattendedCheck = 5 * time.Second
|
|
// chatUnattendedTurnLimit releases an unattended room whose turn never
|
|
// reports an end (a stuck harness or a lost end event).
|
|
chatUnattendedTurnLimit = 6 * time.Hour
|
|
// chatUnattendedRecoverSettle keeps a room recovered at start alive until
|
|
// its freshly attached stream has had time to announce the running turn.
|
|
chatUnattendedRecoverSettle = 15 * time.Second
|
|
// chatRecoverWindow limits start-up probing to sessions used recently.
|
|
chatRecoverWindow = 24 * time.Hour
|
|
)
|
|
|
|
// chatUnattendedTimings is the hub's copy of the timings above, taken when the
|
|
// hub is built so its goroutines never read the package variables.
|
|
type chatUnattendedTimings struct {
|
|
check, turnLimit, settle, window time.Duration
|
|
}
|
|
|
|
func currentChatUnattendedTimings() chatUnattendedTimings {
|
|
return chatUnattendedTimings{check: chatUnattendedCheck, turnLimit: chatUnattendedTurnLimit, settle: chatUnattendedRecoverSettle, window: chatRecoverWindow}
|
|
}
|
|
|
|
// chatTurnProber is implemented by a broker that can ask the harness itself,
|
|
// with I/O, whether a session's turn is in flight. It is used once at server
|
|
// start, when the in-memory turn state is empty.
|
|
type chatTurnProber interface {
|
|
ProbeTurn(ctx context.Context, sessionID string) (bool, error)
|
|
}
|
|
|
|
// keepUnattendedLocked reports whether a room with no client still has work
|
|
// the operator must hear about: a turn in flight (the runner's, or one its
|
|
// stream announced), an unresolved approval or question, or a harness that
|
|
// reports its own turn running (busy, read by the caller off the hub mutex),
|
|
// or background tasks still running.
|
|
// hold honours the start-up settle window. Queued prompts are the caller's
|
|
// concern. Callers hold h.mu.
|
|
func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool {
|
|
if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 {
|
|
return true
|
|
}
|
|
// Running background tasks keep the stream observed so their actual exits
|
|
// are seen and the chat list's indicator clears, without a fixed age cap.
|
|
if h.backgroundTasksLocked(room.sessionID) > 0 {
|
|
return true
|
|
}
|
|
return hold && h.clock().Before(room.holdUntil)
|
|
}
|
|
|
|
// unattendedCountLocked counts the rooms kept observing with no client and no
|
|
// queued prompt, other than except. Callers hold h.mu.
|
|
func (h *chatHub) unattendedCountLocked(except *chatRoom) int {
|
|
n := 0
|
|
for _, room := range h.rooms {
|
|
if room != except && room.watching && len(room.subs) == 0 && len(room.queue) == 0 {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// leaveUnattendedLocked decides the fate of a room whose last client just
|
|
// left: a room holding queued prompts stays for its runner (as before); a room
|
|
// with an in-flight turn or pending attention keeps observing unattended,
|
|
// within the bound; anything else is released. Callers hold h.mu.
|
|
func (h *chatHub) leaveUnattendedLocked(room *chatRoom) {
|
|
if len(room.queue) > 0 {
|
|
h.watchUnattendedLocked(room)
|
|
return
|
|
}
|
|
if !h.keepUnattendedLocked(room, false, true) {
|
|
h.releaseRoomLocked(room)
|
|
return
|
|
}
|
|
if !room.watching && h.backgroundTasksLocked(room.sessionID) == 0 && h.unattendedCountLocked(room) >= chatUnattendedLimit {
|
|
log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit)
|
|
h.releaseRoomLocked(room)
|
|
return
|
|
}
|
|
h.watchUnattendedLocked(room)
|
|
}
|
|
|
|
// watchUnattendedLocked starts the room's unattended watcher once. Callers
|
|
// hold h.mu.
|
|
func (h *chatHub) watchUnattendedLocked(room *chatRoom) {
|
|
if room.watching || room.ctx.Err() != nil {
|
|
return
|
|
}
|
|
room.watching = true
|
|
room.unattendedSince = h.clock()
|
|
go h.watchUnattended(room)
|
|
}
|
|
|
|
func (h *chatHub) watchUnattended(room *chatRoom) {
|
|
check := h.unattended.check
|
|
if check <= 0 {
|
|
check = 5 * time.Second
|
|
}
|
|
ticker := time.NewTicker(check)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-room.ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
if h.reapUnattended(room) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// reapUnattended re-checks one unattended room and reports whether its
|
|
// watcher is finished: a client reattached (the room is attended again), or
|
|
// the room had nothing left to observe, or its turn exceeded the limit, and
|
|
// was released.
|
|
func (h *chatHub) reapUnattended(room *chatRoom) bool {
|
|
h.mu.Lock()
|
|
if done := h.attendedLocked(room); done {
|
|
h.mu.Unlock()
|
|
return true
|
|
}
|
|
broker := room.broker
|
|
h.mu.Unlock()
|
|
// TurnRunning performs no I/O but may take the broker's own lock, so it is
|
|
// never asked under the hub mutex.
|
|
busy := brokerTurnRunning(broker, room.sessionID)
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if done := h.attendedLocked(room); done {
|
|
return true
|
|
}
|
|
if len(room.queue) > 0 {
|
|
// The runner owns a room with queued prompts until they are delivered;
|
|
// the stuck-turn limit counts from the last of them.
|
|
room.unattendedSince = h.clock()
|
|
return false
|
|
}
|
|
// A running task is actual work, not an idle-observer lease. Its terminal
|
|
// lifecycle/readback, or explicit operator stop, releases it normally.
|
|
if h.backgroundTasksLocked(room.sessionID) > 0 {
|
|
return false
|
|
}
|
|
expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit
|
|
if !expired && h.keepUnattendedLocked(room, busy, true) {
|
|
return false
|
|
}
|
|
if expired {
|
|
log.Printf("chat session %s: released the unattended observer after %s without its turn or background tasks ending", room.sessionID, h.unattended.turnLimit)
|
|
}
|
|
room.watching = false
|
|
h.releaseRoomLocked(room)
|
|
return true
|
|
}
|
|
|
|
// attendedLocked reports whether the watcher should stop because the room is
|
|
// gone or a client is watching it again. Callers hold h.mu.
|
|
func (h *chatHub) attendedLocked(room *chatRoom) bool {
|
|
if h.rooms[room.sessionID] != room || room.ctx.Err() != nil {
|
|
room.watching = false
|
|
return true
|
|
}
|
|
if len(room.subs) > 0 {
|
|
room.watching = false
|
|
room.unattendedSince = time.Time{}
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// observeRunningTurns discovers live foreground or background work even when
|
|
// no observer survived long enough to journal it (including a first upgrade or
|
|
// an abrupt old-server exit). Every open registered session is considered; old
|
|
// metadata is not evidence that a long-running task ended. Work is bounded to
|
|
// batches of 32 with four concurrent readback probes, never prompt/create/resume.
|
|
func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
|
|
if h.registry == nil {
|
|
return
|
|
}
|
|
sessions, err := h.registry.OperatorSessions(ctx)
|
|
if err != nil {
|
|
log.Printf("chat recovery registry: %v", err)
|
|
return
|
|
}
|
|
ids := make([]string, 0, len(sessions))
|
|
for _, session := range sessions {
|
|
if session.ClosedAt == 0 && validChatSessionID(session.ID) {
|
|
ids = append(ids, session.ID)
|
|
}
|
|
}
|
|
for start := 0; start < len(ids); start += chatRecoverSessionLimit {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
end := start + chatRecoverSessionLimit
|
|
if end > len(ids) {
|
|
end = len(ids)
|
|
}
|
|
jobs := make(chan string)
|
|
var workers sync.WaitGroup
|
|
for i := 0; i < 4; i++ {
|
|
workers.Add(1)
|
|
go func() {
|
|
defer workers.Done()
|
|
for id := range jobs {
|
|
h.recoverDiscoveredSession(ctx, broker, id)
|
|
}
|
|
}()
|
|
}
|
|
for _, id := range ids[start:end] {
|
|
select {
|
|
case jobs <- id:
|
|
case <-ctx.Done():
|
|
close(jobs)
|
|
workers.Wait()
|
|
return
|
|
}
|
|
}
|
|
close(jobs)
|
|
workers.Wait()
|
|
}
|
|
}
|
|
|
|
// observeUnattended opens (or reuses) the session's room with no client and
|
|
// observes its stream until the turn the harness reported has ended.
|
|
func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
|
|
h.mu.Lock()
|
|
if h.rooms[sessionID] == nil && h.backgroundTasksLocked(sessionID) == 0 && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
|
|
h.mu.Unlock()
|
|
log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit)
|
|
return
|
|
}
|
|
room := h.roomLocked(sessionID)
|
|
if room.broker == nil {
|
|
room.broker = broker
|
|
}
|
|
if len(room.subs) == 0 {
|
|
room.holdUntil = h.clock().Add(h.unattended.settle)
|
|
h.watchUnattendedLocked(room)
|
|
}
|
|
h.mu.Unlock()
|
|
if err := h.ensureObserving(room, broker); err != nil {
|
|
if room.ctx.Err() == nil {
|
|
log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err)
|
|
go h.retryRoomObservation(room, broker)
|
|
}
|
|
return
|
|
}
|
|
h.startRoomPoller(room, broker)
|
|
}
|