316 lines
11 KiB
Go
316 lines
11 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// Chat sessions survive a Pipelines restart. Only an operator action (closing
|
|
// the chat, or Stop on its turn) stops one.
|
|
//
|
|
// Per-backend contract (chatShutdownDetacher and chatSessionReleaser in
|
|
// chat_manager.go). On shutdown chatManager.Close calls Detach on a backend
|
|
// that implements it instead of Close: the backend releases its own connection
|
|
// or process handles but must not interrupt, abort or kill any session, turn,
|
|
// question or background task. On the next start the server reattaches through
|
|
// the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still
|
|
// running?), Stream (re-subscribe), Permissions/Forms (pending questions and
|
|
// approvals), Shells (background tasks) and Messages (read back a turn that
|
|
// ended while the server was down). chat_close calls ReleaseSession on a
|
|
// backend that keeps an agent alive until released. A backend without Detach
|
|
// is closed as before, and its sessions end with the server.
|
|
//
|
|
// OpenCode (an attached or detached owned service), Codex (its App Server) and
|
|
// Claude Code (its detachable hosts) implement Detach.
|
|
|
|
// chatLiveStore durably journals live work before room eviction/crash, as well
|
|
// as on graceful shutdown; *pipeline.Store implements it.
|
|
type chatLiveStore interface {
|
|
ChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
|
|
RememberChatLiveSession(ctx context.Context, session pipeline.ChatLiveSession) error
|
|
ForgetChatLiveSession(ctx context.Context, id string) error
|
|
}
|
|
|
|
// chatReattachPoll is how often a reattached turn re-asks its harness whether
|
|
// it is still running, to fold in an end the re-subscribed stream did not see.
|
|
var chatReattachPoll = 2 * time.Second
|
|
|
|
// detachSessions records every chat with work still live on its harness: a
|
|
// turn in flight, a pending question or approval, or running background tasks.
|
|
// It must run before rooms are torn down (before the server context is
|
|
// canceled), and it never touches the sessions themselves.
|
|
func (h *chatHub) detachSessions(ctx context.Context) error {
|
|
if h.liveStore == nil {
|
|
return nil
|
|
}
|
|
h.mu.Lock()
|
|
var live []pipeline.ChatLiveSession
|
|
for id, room := range h.rooms {
|
|
turn := room.running || room.observedTurn
|
|
if step := h.turns[id]; step != nil && step.step != nil && step.step.Status == protocol.ChatStepStarted && !step.step.Intermediate {
|
|
turn = true
|
|
}
|
|
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
|
|
if turn || attention {
|
|
live = append(live, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: room.nativeTurnID, Attention: attention, RecordedAt: h.clock()})
|
|
}
|
|
}
|
|
h.mu.Unlock()
|
|
if len(live) > 0 {
|
|
log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live))
|
|
}
|
|
for _, session := range live {
|
|
if err := h.liveStore.RememberChatLiveSession(ctx, session); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// reattachSessions reads (without consuming) the recovery journal. Each chat
|
|
// is observed again (stream, pending questions, background tasks) while its
|
|
// harness still has work, and a turn that ended while the server was down is
|
|
// folded in from the harness's history, so its reply still notifies and its
|
|
// queued prompts advance, as if it had been observed live.
|
|
func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
|
|
if h.liveStore == nil || broker == nil {
|
|
return
|
|
}
|
|
live, err := h.liveStore.ChatLiveSessions(ctx)
|
|
if err != nil {
|
|
log.Printf("chat reattach: %v", err)
|
|
return
|
|
}
|
|
for _, session := range live {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
if !validChatSessionID(session.SessionID) {
|
|
continue
|
|
}
|
|
h.mu.Lock()
|
|
observed := h.rooms[session.SessionID] != nil
|
|
generation := h.recoveryGeneration[session.SessionID]
|
|
h.mu.Unlock()
|
|
if observed {
|
|
continue
|
|
}
|
|
if h.registry != nil {
|
|
row, found, err := h.registry.OperatorSession(ctx, session.SessionID)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if !found || row.ClosedAt != 0 {
|
|
h.forgetLive(ctx, session.SessionID, generation)
|
|
continue
|
|
}
|
|
}
|
|
h.reattachSession(ctx, broker, session)
|
|
}
|
|
}
|
|
|
|
func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) {
|
|
id := session.SessionID
|
|
h.mu.Lock()
|
|
generation := h.recoveryGeneration[id]
|
|
h.mu.Unlock()
|
|
running := false
|
|
turnKnown := false
|
|
if prober, ok := broker.(chatTurnProber); ok {
|
|
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
busy, err := prober.ProbeTurn(probeCtx, id)
|
|
cancel()
|
|
if err != nil {
|
|
// Unknown is not idle: keep observing rather than guess it ended.
|
|
log.Printf("chat reattach probe session=%s failed (%T)", id, err)
|
|
busy = session.Turn
|
|
}
|
|
running = busy
|
|
turnKnown = err == nil
|
|
}
|
|
var snapshot []protocol.ChatShell
|
|
attentionKnown := false
|
|
if !running {
|
|
// A recovered task may have ended while disconnected. Only clear the
|
|
// journal after all pending-work surfaces authoritatively report idle.
|
|
if attention, known, shells := probeChatAttention(ctx, broker, id); known {
|
|
snapshot = shells
|
|
attentionKnown = true
|
|
session.Attention = attention
|
|
} else {
|
|
session.Attention = true // Unknown pending work is not idle.
|
|
}
|
|
}
|
|
h.mu.Lock()
|
|
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
|
|
if !stale && attentionKnown {
|
|
h.replaceShellsLocked(id, snapshot)
|
|
}
|
|
h.mu.Unlock()
|
|
if stale {
|
|
return
|
|
}
|
|
if turnKnown && !running && !session.Attention {
|
|
if session.Turn {
|
|
h.foldEndedTurn(ctx, broker, session, nil)
|
|
}
|
|
if h.liveStore != nil {
|
|
h.forgetLive(ctx, id, generation)
|
|
}
|
|
return
|
|
}
|
|
retained := session
|
|
retained.Turn = running
|
|
retained.RecordedAt = h.clock()
|
|
if !h.rememberRecovered(ctx, retained, generation) {
|
|
return
|
|
}
|
|
if running && session.Turn {
|
|
// Seed ownership before opening the stream: replay/poll admission can
|
|
// journal immediately and must not overwrite the recovered running turn.
|
|
h.mu.Lock()
|
|
turn := h.turnLocked(id)
|
|
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
|
|
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
|
|
}
|
|
h.mu.Unlock()
|
|
}
|
|
if running || session.Attention {
|
|
h.observeUnattended(id, broker)
|
|
}
|
|
if !session.Turn {
|
|
return
|
|
}
|
|
if !running {
|
|
h.foldEndedTurn(ctx, broker, session, nil)
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
room := h.rooms[id]
|
|
if room == nil {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
// The turn the previous process watched is still running: hold the queue
|
|
// behind it and remember it as started, so its end notifies as usual.
|
|
turn := h.turnLocked(id)
|
|
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
|
|
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
|
|
}
|
|
if room.nativeTurnID == "" {
|
|
room.nativeTurnID = session.NativeTurnID
|
|
}
|
|
room.observedTurn = true
|
|
h.mu.Unlock()
|
|
go h.awaitReattachedTurn(room, broker, session)
|
|
}
|
|
|
|
// awaitReattachedTurn waits for a reattached turn to end. The stream normally
|
|
// reports it; when the harness reports the turn idle without the stream having
|
|
// said so, the end is folded in from history.
|
|
func (h *chatHub) awaitReattachedTurn(room *chatRoom, broker chatBroker, session pipeline.ChatLiveSession) {
|
|
prober, _ := broker.(chatTurnProber)
|
|
started := h.clock()
|
|
ticker := time.NewTicker(chatReattachPoll)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-room.ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
h.mu.Lock()
|
|
ended := !room.observedTurn || h.rooms[room.sessionID] != room
|
|
h.mu.Unlock()
|
|
if ended || prober == nil || h.clock().Sub(started) >= h.unattended.turnLimit {
|
|
return
|
|
}
|
|
probeCtx, cancel := context.WithTimeout(room.ctx, 5*time.Second)
|
|
busy, err := prober.ProbeTurn(probeCtx, room.sessionID)
|
|
cancel()
|
|
if err != nil || busy {
|
|
continue
|
|
}
|
|
h.foldEndedTurn(room.ctx, broker, session, room)
|
|
return
|
|
}
|
|
}
|
|
|
|
// foldEndedTurn settles a turn that ended without the hub observing its end:
|
|
// attached clients are told to re-read the authoritative history, the room's
|
|
// turn slot is released (so queued prompts are delivered) and the reply is
|
|
// announced once, with its text read back from the harness.
|
|
func (h *chatHub) foldEndedTurn(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession, room *chatRoom) {
|
|
id := session.SessionID
|
|
readCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
page, err := broker.Messages(readCtx, id, 20, "")
|
|
cancel()
|
|
if err != nil {
|
|
log.Printf("chat reattach history session=%s failed (%T)", id, err)
|
|
}
|
|
replyID, body := lastReply(page.Messages)
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if room != nil && h.rooms[id] == room && room.observedTurn {
|
|
end := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: room.nativeTurnID}}
|
|
h.applyLocked(end)
|
|
h.finishTurnLocked(room)
|
|
resync := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventResync}
|
|
for _, sub := range room.subs {
|
|
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &resync})
|
|
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &end})
|
|
}
|
|
if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) {
|
|
room.watching = false
|
|
h.releaseRoomLocked(room)
|
|
}
|
|
}
|
|
if replyID == "" || !h.attentionWanted() {
|
|
return
|
|
}
|
|
key := session.NativeTurnID
|
|
if key == "" {
|
|
key = replyID
|
|
}
|
|
if !h.claimChatAttentionLocked("reply\x00" + id + "\x00" + key) {
|
|
return
|
|
}
|
|
if body == "" {
|
|
body = chatReplyFallbackBody
|
|
}
|
|
log.Printf("chat session %s: turn finished while the server was restarting; reply read back from history", id)
|
|
h.startChatAttention(chatAttention{sessionID: id, reply: true, body: body, id: chatNotificationID(id, "reply", key)})
|
|
}
|
|
|
|
// lastReply returns the id and bounded text of the assistant reply that ends
|
|
// a history page (messages after the last user message), or "" when the page
|
|
// ends with the user's own message.
|
|
func lastReply(messages []protocol.ChatMessage) (string, string) {
|
|
for i := len(messages) - 1; i >= 0; i-- {
|
|
message := messages[i]
|
|
if message.Role == protocol.ChatRoleUser {
|
|
return "", ""
|
|
}
|
|
if message.Kind != "" {
|
|
continue
|
|
}
|
|
var text strings.Builder
|
|
for _, part := range message.Parts {
|
|
if part.Type != protocol.ChatPartText || part.Text == "" {
|
|
continue
|
|
}
|
|
if text.Len() > 0 {
|
|
text.WriteString("\n\n")
|
|
}
|
|
text.WriteString(part.Text)
|
|
}
|
|
return message.ID, truncateRunes(paragraphs(text.String()), chatReplyBodyRunes)
|
|
}
|
|
return "", ""
|
|
}
|