a627b38ef3
OpenCode and Codex chats implement Detach (called by chatManager.Close) instead of a parallel DetachSessions interface. Docs follow the Claude Code host and the systemd KillMode=process unit. The opt-in live restart check copies only the stage provider's login row into its scratch OpenCode database, builds the CLI for skill install and removes its scratch tree.
241 lines
8.6 KiB
Go
241 lines
8.6 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// Chat sessions survive a Pipelines restart. Only an operator action (closing
|
|
// the chat, or Stop on its turn) stops one.
|
|
//
|
|
// Per-backend contract (chatShutdownDetacher and chatSessionReleaser in
|
|
// chat_manager.go). On shutdown chatManager.Close calls Detach on a backend
|
|
// that implements it instead of Close: the backend releases its own connection
|
|
// or process handles but must not interrupt, abort or kill any session, turn,
|
|
// question or background task. On the next start the server reattaches through
|
|
// the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still
|
|
// running?), Stream (re-subscribe), Permissions/Forms (pending questions and
|
|
// approvals), Shells (background tasks) and Messages (read back a turn that
|
|
// ended while the server was down). chat_close calls ReleaseSession on a
|
|
// backend that keeps an agent alive until released. A backend without Detach
|
|
// is closed as before, and its sessions end with the server.
|
|
//
|
|
// OpenCode (an attached or detached owned service), Codex (its App Server) and
|
|
// Claude Code (its detachable hosts) implement Detach.
|
|
|
|
// chatLiveStore records, on shutdown, the chats left live on their harness so
|
|
// the next start reattaches exactly those; *pipeline.Store implements it.
|
|
type chatLiveStore interface {
|
|
RecordChatLiveSessions(ctx context.Context, sessions []pipeline.ChatLiveSession) error
|
|
TakeChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
|
|
}
|
|
|
|
// chatReattachPoll is how often a reattached turn re-asks its harness whether
|
|
// it is still running, to fold in an end the re-subscribed stream did not see.
|
|
var chatReattachPoll = 2 * time.Second
|
|
|
|
// detachSessions records every chat with work still live on its harness: a
|
|
// turn in flight, a pending question or approval, or running background tasks.
|
|
// It must run before rooms are torn down (before the server context is
|
|
// canceled), and it never touches the sessions themselves.
|
|
func (h *chatHub) detachSessions(ctx context.Context) error {
|
|
if h.liveStore == nil {
|
|
return nil
|
|
}
|
|
h.mu.Lock()
|
|
var live []pipeline.ChatLiveSession
|
|
for id, room := range h.rooms {
|
|
turn := room.running || room.observedTurn
|
|
if step := h.turns[id]; step != nil && step.step != nil && step.step.Status == protocol.ChatStepStarted && !step.step.Intermediate {
|
|
turn = true
|
|
}
|
|
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
|
|
if turn || attention {
|
|
live = append(live, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: room.nativeTurnID, Attention: attention, RecordedAt: h.clock()})
|
|
}
|
|
}
|
|
h.mu.Unlock()
|
|
if len(live) > 0 {
|
|
log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live))
|
|
}
|
|
return h.liveStore.RecordChatLiveSessions(ctx, live)
|
|
}
|
|
|
|
// reattachSessions is the start-up half of detachSessions: each recorded chat
|
|
// is observed again (stream, pending questions, background tasks) while its
|
|
// harness still has work, and a turn that ended while the server was down is
|
|
// folded in from the harness's history, so its reply still notifies and its
|
|
// queued prompts advance, as if it had been observed live.
|
|
func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
|
|
if h.liveStore == nil || broker == nil {
|
|
return
|
|
}
|
|
live, err := h.liveStore.TakeChatLiveSessions(ctx)
|
|
if err != nil {
|
|
log.Printf("chat reattach: %v", err)
|
|
return
|
|
}
|
|
for _, session := range live {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
if !validChatSessionID(session.SessionID) {
|
|
continue
|
|
}
|
|
h.reattachSession(ctx, broker, session)
|
|
}
|
|
}
|
|
|
|
func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) {
|
|
id := session.SessionID
|
|
running := false
|
|
if prober, ok := broker.(chatTurnProber); ok {
|
|
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
busy, err := prober.ProbeTurn(probeCtx, id)
|
|
cancel()
|
|
if err != nil {
|
|
// Unknown is not idle: keep observing rather than guess it ended.
|
|
log.Printf("chat reattach probe session=%s failed (%T)", id, err)
|
|
busy = session.Turn
|
|
}
|
|
running = busy
|
|
}
|
|
if running || session.Attention {
|
|
h.observeUnattended(id, broker)
|
|
}
|
|
if !session.Turn {
|
|
return
|
|
}
|
|
if !running {
|
|
h.foldEndedTurn(ctx, broker, session, nil)
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
room := h.rooms[id]
|
|
if room == nil {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
// The turn the previous process watched is still running: hold the queue
|
|
// behind it and remember it as started, so its end notifies as usual.
|
|
turn := h.turnLocked(id)
|
|
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
|
|
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
|
|
}
|
|
if room.nativeTurnID == "" {
|
|
room.nativeTurnID = session.NativeTurnID
|
|
}
|
|
room.observedTurn = true
|
|
h.mu.Unlock()
|
|
go h.awaitReattachedTurn(room, broker, session)
|
|
}
|
|
|
|
// awaitReattachedTurn waits for a reattached turn to end. The stream normally
|
|
// reports it; when the harness reports the turn idle without the stream having
|
|
// said so, the end is folded in from history.
|
|
func (h *chatHub) awaitReattachedTurn(room *chatRoom, broker chatBroker, session pipeline.ChatLiveSession) {
|
|
prober, _ := broker.(chatTurnProber)
|
|
started := h.clock()
|
|
ticker := time.NewTicker(chatReattachPoll)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-room.ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
h.mu.Lock()
|
|
ended := !room.observedTurn || h.rooms[room.sessionID] != room
|
|
h.mu.Unlock()
|
|
if ended || prober == nil || h.clock().Sub(started) >= h.unattended.turnLimit {
|
|
return
|
|
}
|
|
probeCtx, cancel := context.WithTimeout(room.ctx, 5*time.Second)
|
|
busy, err := prober.ProbeTurn(probeCtx, room.sessionID)
|
|
cancel()
|
|
if err != nil || busy {
|
|
continue
|
|
}
|
|
h.foldEndedTurn(room.ctx, broker, session, room)
|
|
return
|
|
}
|
|
}
|
|
|
|
// foldEndedTurn settles a turn that ended without the hub observing its end:
|
|
// attached clients are told to re-read the authoritative history, the room's
|
|
// turn slot is released (so queued prompts are delivered) and the reply is
|
|
// announced once, with its text read back from the harness.
|
|
func (h *chatHub) foldEndedTurn(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession, room *chatRoom) {
|
|
id := session.SessionID
|
|
readCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
page, err := broker.Messages(readCtx, id, 20, "")
|
|
cancel()
|
|
if err != nil {
|
|
log.Printf("chat reattach history session=%s failed (%T)", id, err)
|
|
}
|
|
replyID, body := lastReply(page.Messages)
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if room != nil && h.rooms[id] == room && room.observedTurn {
|
|
end := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: room.nativeTurnID}}
|
|
h.applyLocked(end)
|
|
h.finishTurnLocked(room)
|
|
resync := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventResync}
|
|
for _, sub := range room.subs {
|
|
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &resync})
|
|
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &end})
|
|
}
|
|
if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) {
|
|
room.watching = false
|
|
h.releaseRoomLocked(room)
|
|
}
|
|
}
|
|
if replyID == "" || !h.attentionWanted() {
|
|
return
|
|
}
|
|
key := session.NativeTurnID
|
|
if key == "" {
|
|
key = replyID
|
|
}
|
|
if !h.claimChatAttentionLocked("reply\x00" + id + "\x00" + key) {
|
|
return
|
|
}
|
|
if body == "" {
|
|
body = chatReplyFallbackBody
|
|
}
|
|
log.Printf("chat session %s: turn finished while the server was restarting; reply read back from history", id)
|
|
h.startChatAttention(chatAttention{sessionID: id, reply: true, body: body, id: chatNotificationID(id, "reply", key)})
|
|
}
|
|
|
|
// lastReply returns the id and bounded text of the assistant reply that ends
|
|
// a history page (messages after the last user message), or "" when the page
|
|
// ends with the user's own message.
|
|
func lastReply(messages []protocol.ChatMessage) (string, string) {
|
|
for i := len(messages) - 1; i >= 0; i-- {
|
|
message := messages[i]
|
|
if message.Role == protocol.ChatRoleUser {
|
|
return "", ""
|
|
}
|
|
if message.Kind != "" {
|
|
continue
|
|
}
|
|
var text strings.Builder
|
|
for _, part := range message.Parts {
|
|
if part.Type != protocol.ChatPartText || part.Text == "" {
|
|
continue
|
|
}
|
|
if text.Len() > 0 {
|
|
text.WriteString("\n\n")
|
|
}
|
|
text.WriteString(part.Text)
|
|
}
|
|
return message.ID, truncateRunes(paragraphs(text.String()), chatReplyBodyRunes)
|
|
}
|
|
return "", ""
|
|
}
|