Files
pipelines/internal/server/chat_detach.go
T

316 lines
11 KiB
Go

package server
import (
"context"
"log"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Chat sessions survive a Pipelines restart. Only an operator action (closing
// the chat, or Stop on its turn) stops one.
//
// Per-backend contract (chatShutdownDetacher and chatSessionReleaser in
// chat_manager.go). On shutdown chatManager.Close calls Detach on a backend
// that implements it instead of Close: the backend releases its own connection
// or process handles but must not interrupt, abort or kill any session, turn,
// question or background task. On the next start the server reattaches through
// the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still
// running?), Stream (re-subscribe), Permissions/Forms (pending questions and
// approvals), Shells (background tasks) and Messages (read back a turn that
// ended while the server was down). chat_close calls ReleaseSession on a
// backend that keeps an agent alive until released. A backend without Detach
// is closed as before, and its sessions end with the server.
//
// OpenCode (an attached or detached owned service), Codex (its App Server) and
// Claude Code (its detachable hosts) implement Detach.
// chatLiveStore durably journals live work before room eviction/crash, as well
// as on graceful shutdown; *pipeline.Store implements it.
type chatLiveStore interface {
ChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
RememberChatLiveSession(ctx context.Context, session pipeline.ChatLiveSession) error
ForgetChatLiveSession(ctx context.Context, id string) error
}
// chatReattachPoll is how often a reattached turn re-asks its harness whether
// it is still running, to fold in an end the re-subscribed stream did not see.
var chatReattachPoll = 2 * time.Second
// detachSessions records every chat with work still live on its harness: a
// turn in flight, a pending question or approval, or running background tasks.
// It must run before rooms are torn down (before the server context is
// canceled), and it never touches the sessions themselves.
func (h *chatHub) detachSessions(ctx context.Context) error {
if h.liveStore == nil {
return nil
}
h.mu.Lock()
var live []pipeline.ChatLiveSession
for id, room := range h.rooms {
turn := room.running || room.observedTurn
if step := h.turns[id]; step != nil && step.step != nil && step.step.Status == protocol.ChatStepStarted && !step.step.Intermediate {
turn = true
}
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
if turn || attention {
live = append(live, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: room.nativeTurnID, Attention: attention, RecordedAt: h.clock()})
}
}
h.mu.Unlock()
if len(live) > 0 {
log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live))
}
for _, session := range live {
if err := h.liveStore.RememberChatLiveSession(ctx, session); err != nil {
return err
}
}
return nil
}
// reattachSessions reads (without consuming) the recovery journal. Each chat
// is observed again (stream, pending questions, background tasks) while its
// harness still has work, and a turn that ended while the server was down is
// folded in from the harness's history, so its reply still notifies and its
// queued prompts advance, as if it had been observed live.
func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
if h.liveStore == nil || broker == nil {
return
}
live, err := h.liveStore.ChatLiveSessions(ctx)
if err != nil {
log.Printf("chat reattach: %v", err)
return
}
for _, session := range live {
if ctx.Err() != nil {
return
}
if !validChatSessionID(session.SessionID) {
continue
}
h.mu.Lock()
observed := h.rooms[session.SessionID] != nil
generation := h.recoveryGeneration[session.SessionID]
h.mu.Unlock()
if observed {
continue
}
if h.registry != nil {
row, found, err := h.registry.OperatorSession(ctx, session.SessionID)
if err != nil {
continue
}
if !found || row.ClosedAt != 0 {
h.forgetLive(ctx, session.SessionID, generation)
continue
}
}
h.reattachSession(ctx, broker, session)
}
}
func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) {
id := session.SessionID
h.mu.Lock()
generation := h.recoveryGeneration[id]
h.mu.Unlock()
running := false
turnKnown := false
if prober, ok := broker.(chatTurnProber); ok {
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
busy, err := prober.ProbeTurn(probeCtx, id)
cancel()
if err != nil {
// Unknown is not idle: keep observing rather than guess it ended.
log.Printf("chat reattach probe session=%s failed (%T)", id, err)
busy = session.Turn
}
running = busy
turnKnown = err == nil
}
var snapshot []protocol.ChatShell
attentionKnown := false
if !running {
// A recovered task may have ended while disconnected. Only clear the
// journal after all pending-work surfaces authoritatively report idle.
if attention, known, shells := probeChatAttention(ctx, broker, id); known {
snapshot = shells
attentionKnown = true
session.Attention = attention
} else {
session.Attention = true // Unknown pending work is not idle.
}
}
h.mu.Lock()
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
if !stale && attentionKnown {
h.replaceShellsLocked(id, snapshot)
}
h.mu.Unlock()
if stale {
return
}
if turnKnown && !running && !session.Attention {
if session.Turn {
h.foldEndedTurn(ctx, broker, session, nil)
}
if h.liveStore != nil {
h.forgetLive(ctx, id, generation)
}
return
}
retained := session
retained.Turn = running
retained.RecordedAt = h.clock()
if !h.rememberRecovered(ctx, retained, generation) {
return
}
if running && session.Turn {
// Seed ownership before opening the stream: replay/poll admission can
// journal immediately and must not overwrite the recovered running turn.
h.mu.Lock()
turn := h.turnLocked(id)
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
}
h.mu.Unlock()
}
if running || session.Attention {
h.observeUnattended(id, broker)
}
if !session.Turn {
return
}
if !running {
h.foldEndedTurn(ctx, broker, session, nil)
return
}
h.mu.Lock()
room := h.rooms[id]
if room == nil {
h.mu.Unlock()
return
}
// The turn the previous process watched is still running: hold the queue
// behind it and remember it as started, so its end notifies as usual.
turn := h.turnLocked(id)
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
}
if room.nativeTurnID == "" {
room.nativeTurnID = session.NativeTurnID
}
room.observedTurn = true
h.mu.Unlock()
go h.awaitReattachedTurn(room, broker, session)
}
// awaitReattachedTurn waits for a reattached turn to end. The stream normally
// reports it; when the harness reports the turn idle without the stream having
// said so, the end is folded in from history.
func (h *chatHub) awaitReattachedTurn(room *chatRoom, broker chatBroker, session pipeline.ChatLiveSession) {
prober, _ := broker.(chatTurnProber)
started := h.clock()
ticker := time.NewTicker(chatReattachPoll)
defer ticker.Stop()
for {
select {
case <-room.ctx.Done():
return
case <-ticker.C:
}
h.mu.Lock()
ended := !room.observedTurn || h.rooms[room.sessionID] != room
h.mu.Unlock()
if ended || prober == nil || h.clock().Sub(started) >= h.unattended.turnLimit {
return
}
probeCtx, cancel := context.WithTimeout(room.ctx, 5*time.Second)
busy, err := prober.ProbeTurn(probeCtx, room.sessionID)
cancel()
if err != nil || busy {
continue
}
h.foldEndedTurn(room.ctx, broker, session, room)
return
}
}
// foldEndedTurn settles a turn that ended without the hub observing its end:
// attached clients are told to re-read the authoritative history, the room's
// turn slot is released (so queued prompts are delivered) and the reply is
// announced once, with its text read back from the harness.
func (h *chatHub) foldEndedTurn(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession, room *chatRoom) {
id := session.SessionID
readCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
page, err := broker.Messages(readCtx, id, 20, "")
cancel()
if err != nil {
log.Printf("chat reattach history session=%s failed (%T)", id, err)
}
replyID, body := lastReply(page.Messages)
h.mu.Lock()
defer h.mu.Unlock()
if room != nil && h.rooms[id] == room && room.observedTurn {
end := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: room.nativeTurnID}}
h.applyLocked(end)
h.finishTurnLocked(room)
resync := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventResync}
for _, sub := range room.subs {
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &resync})
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &end})
}
if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) {
room.watching = false
h.releaseRoomLocked(room)
}
}
if replyID == "" || !h.attentionWanted() {
return
}
key := session.NativeTurnID
if key == "" {
key = replyID
}
if !h.claimChatAttentionLocked("reply\x00" + id + "\x00" + key) {
return
}
if body == "" {
body = chatReplyFallbackBody
}
log.Printf("chat session %s: turn finished while the server was restarting; reply read back from history", id)
h.startChatAttention(chatAttention{sessionID: id, reply: true, body: body, id: chatNotificationID(id, "reply", key)})
}
// lastReply returns the id and bounded text of the assistant reply that ends
// a history page (messages after the last user message), or "" when the page
// ends with the user's own message.
func lastReply(messages []protocol.ChatMessage) (string, string) {
for i := len(messages) - 1; i >= 0; i-- {
message := messages[i]
if message.Role == protocol.ChatRoleUser {
return "", ""
}
if message.Kind != "" {
continue
}
var text strings.Builder
for _, part := range message.Parts {
if part.Type != protocol.ChatPartText || part.Text == "" {
continue
}
if text.Len() > 0 {
text.WriteString("\n\n")
}
text.WriteString(part.Text)
}
return message.ID, truncateRunes(paragraphs(text.String()), chatReplyBodyRunes)
}
return "", ""
}