package server import ( "context" "log" "strings" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" ) // Chat sessions survive a Pipelines restart. Only an operator action (closing // the chat, or Stop on its turn) stops one. // // Per-backend contract (chatShutdownDetacher and chatSessionReleaser in // chat_manager.go). On shutdown chatManager.Close calls Detach on a backend // that implements it instead of Close: the backend releases its own connection // or process handles but must not interrupt, abort or kill any session, turn, // question or background task. On the next start the server reattaches through // the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still // running?), Stream (re-subscribe), Permissions/Forms (pending questions and // approvals), Shells (background tasks) and Messages (read back a turn that // ended while the server was down). chat_close calls ReleaseSession on a // backend that keeps an agent alive until released. A backend without Detach // is closed as before, and its sessions end with the server. // // OpenCode (an attached or detached owned service), Codex (its App Server) and // Claude Code (its detachable hosts) implement Detach. // chatLiveStore durably journals live work before room eviction/crash, as well // as on graceful shutdown; *pipeline.Store implements it. type chatLiveStore interface { ChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error) RememberChatLiveSession(ctx context.Context, session pipeline.ChatLiveSession) error ForgetChatLiveSession(ctx context.Context, id string) error } // chatReattachPoll is how often a reattached turn re-asks its harness whether // it is still running, to fold in an end the re-subscribed stream did not see. var chatReattachPoll = 2 * time.Second // detachSessions records every chat with work still live on its harness: a // turn in flight, a pending question or approval, or running background tasks. // It must run before rooms are torn down (before the server context is // canceled), and it never touches the sessions themselves. func (h *chatHub) detachSessions(ctx context.Context) error { if h.liveStore == nil { return nil } h.mu.Lock() var live []pipeline.ChatLiveSession for id, room := range h.rooms { turn := room.running || room.observedTurn if step := h.turns[id]; step != nil && step.step != nil && step.step.Status == protocol.ChatStepStarted && !step.step.Intermediate { turn = true } attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0 if turn || attention { live = append(live, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: room.nativeTurnID, Attention: attention, RecordedAt: h.clock()}) } } h.mu.Unlock() if len(live) > 0 { log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live)) } for _, session := range live { if err := h.liveStore.RememberChatLiveSession(ctx, session); err != nil { return err } } return nil } // reattachSessions reads (without consuming) the recovery journal. Each chat // is observed again (stream, pending questions, background tasks) while its // harness still has work, and a turn that ended while the server was down is // folded in from the harness's history, so its reply still notifies and its // queued prompts advance, as if it had been observed live. func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) { if h.liveStore == nil || broker == nil { return } live, err := h.liveStore.ChatLiveSessions(ctx) if err != nil { log.Printf("chat reattach: %v", err) return } for _, session := range live { if ctx.Err() != nil { return } if !validChatSessionID(session.SessionID) { continue } h.mu.Lock() observed := h.rooms[session.SessionID] != nil generation := h.recoveryGeneration[session.SessionID] h.mu.Unlock() if observed { continue } if h.registry != nil { row, found, err := h.registry.OperatorSession(ctx, session.SessionID) if err != nil { continue } if !found || row.ClosedAt != 0 { h.forgetLive(ctx, session.SessionID, generation) continue } } h.reattachSession(ctx, broker, session) } } func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) { id := session.SessionID h.mu.Lock() generation := h.recoveryGeneration[id] h.mu.Unlock() running := false turnKnown := false if prober, ok := broker.(chatTurnProber); ok { probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) busy, err := prober.ProbeTurn(probeCtx, id) cancel() if err != nil { // Unknown is not idle: keep observing rather than guess it ended. log.Printf("chat reattach probe session=%s failed (%T)", id, err) busy = session.Turn } running = busy turnKnown = err == nil } var snapshot []protocol.ChatShell attentionKnown := false if !running { // A recovered task may have ended while disconnected. Only clear the // journal after all pending-work surfaces authoritatively report idle. if attention, known, shells := probeChatAttention(ctx, broker, id); known { snapshot = shells attentionKnown = true session.Attention = attention } else { session.Attention = true // Unknown pending work is not idle. } } h.mu.Lock() stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil if !stale && attentionKnown { h.replaceShellsLocked(id, snapshot) } h.mu.Unlock() if stale { return } if turnKnown && !running && !session.Attention { if session.Turn { h.foldEndedTurn(ctx, broker, session, nil) } if h.liveStore != nil { h.forgetLive(ctx, id, generation) } return } retained := session retained.Turn = running retained.RecordedAt = h.clock() if !h.rememberRecovered(ctx, retained, generation) { return } if running && session.Turn { // Seed ownership before opening the stream: replay/poll admission can // journal immediately and must not overwrite the recovered running turn. h.mu.Lock() turn := h.turnLocked(id) if turn.step == nil || turn.step.Status == protocol.ChatStepEnded { turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID} } h.mu.Unlock() } if running || session.Attention { h.observeUnattended(id, broker) } if !session.Turn { return } if !running { h.foldEndedTurn(ctx, broker, session, nil) return } h.mu.Lock() room := h.rooms[id] if room == nil { h.mu.Unlock() return } // The turn the previous process watched is still running: hold the queue // behind it and remember it as started, so its end notifies as usual. turn := h.turnLocked(id) if turn.step == nil || turn.step.Status == protocol.ChatStepEnded { turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID} } if room.nativeTurnID == "" { room.nativeTurnID = session.NativeTurnID } room.observedTurn = true h.mu.Unlock() go h.awaitReattachedTurn(room, broker, session) } // awaitReattachedTurn waits for a reattached turn to end. The stream normally // reports it; when the harness reports the turn idle without the stream having // said so, the end is folded in from history. func (h *chatHub) awaitReattachedTurn(room *chatRoom, broker chatBroker, session pipeline.ChatLiveSession) { prober, _ := broker.(chatTurnProber) started := h.clock() ticker := time.NewTicker(chatReattachPoll) defer ticker.Stop() for { select { case <-room.ctx.Done(): return case <-ticker.C: } h.mu.Lock() ended := !room.observedTurn || h.rooms[room.sessionID] != room h.mu.Unlock() if ended || prober == nil || h.clock().Sub(started) >= h.unattended.turnLimit { return } probeCtx, cancel := context.WithTimeout(room.ctx, 5*time.Second) busy, err := prober.ProbeTurn(probeCtx, room.sessionID) cancel() if err != nil || busy { continue } h.foldEndedTurn(room.ctx, broker, session, room) return } } // foldEndedTurn settles a turn that ended without the hub observing its end: // attached clients are told to re-read the authoritative history, the room's // turn slot is released (so queued prompts are delivered) and the reply is // announced once, with its text read back from the harness. func (h *chatHub) foldEndedTurn(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession, room *chatRoom) { id := session.SessionID readCtx, cancel := context.WithTimeout(ctx, 10*time.Second) page, err := broker.Messages(readCtx, id, 20, "") cancel() if err != nil { log.Printf("chat reattach history session=%s failed (%T)", id, err) } replyID, body := lastReply(page.Messages) h.mu.Lock() defer h.mu.Unlock() if room != nil && h.rooms[id] == room && room.observedTurn { end := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: room.nativeTurnID}} h.applyLocked(end) h.finishTurnLocked(room) resync := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventResync} for _, sub := range room.subs { enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &resync}) enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &end}) } if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) { room.watching = false h.releaseRoomLocked(room) } } if replyID == "" || !h.attentionWanted() { return } key := session.NativeTurnID if key == "" { key = replyID } if !h.claimChatAttentionLocked("reply\x00" + id + "\x00" + key) { return } if body == "" { body = chatReplyFallbackBody } log.Printf("chat session %s: turn finished while the server was restarting; reply read back from history", id) h.startChatAttention(chatAttention{sessionID: id, reply: true, body: body, id: chatNotificationID(id, "reply", key)}) } // lastReply returns the id and bounded text of the assistant reply that ends // a history page (messages after the last user message), or "" when the page // ends with the user's own message. func lastReply(messages []protocol.ChatMessage) (string, string) { for i := len(messages) - 1; i >= 0; i-- { message := messages[i] if message.Role == protocol.ChatRoleUser { return "", "" } if message.Kind != "" { continue } var text strings.Builder for _, part := range message.Parts { if part.Type != protocol.ChatPartText || part.Text == "" { continue } if text.Len() > 0 { text.WriteString("\n\n") } text.WriteString(part.Text) } return message.ID, truncateRunes(paragraphs(text.String()), chatReplyBodyRunes) } return "", "" }