Files
pipelines/internal/server/chat_recovery_journal.go
T
alan 2307d45417 Release native chat runtimes on lifecycle changes
Ensure /new and chat close stop native foreground and background work before changing registry state. Preserve transcripts while surfacing release and recovery cleanup failures instead of reporting false success, with focused adapter and server regressions.
2026-11-11 17:11:40 +00:00

210 lines
6.2 KiB
Go

package server
import (
"context"
"log"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type chatRecoveryState struct {
turn, attention bool
native string
}
// rememberLiveLocked journals liveness before an observer can be evicted or
// the process can crash. A bounded replay cache is never the recovery journal.
// Callers hold h.mu; like durable prompt admission, this small SQLite write is
// ordered with the lifecycle change it describes.
func (h *chatHub) rememberLiveLocked(id string) {
if h.liveStore == nil || (h.baseCtx != nil && h.baseCtx.Err() != nil) {
return
}
room := h.rooms[id]
if room == nil {
return
} // Losing observation is not authoritative idle.
turn := room.running || room.observedTurn
native := room.nativeTurnID
if current := h.turns[id]; current != nil && current.step != nil && current.step.Status == protocol.ChatStepStarted && !current.step.Intermediate {
turn = true
if native == "" {
native = current.step.TurnID
}
}
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
next := chatRecoveryState{turn: turn, attention: attention, native: native}
if previous, ok := h.liveJournal[id]; ok && previous == next {
return
}
h.touchRecoveryLocked(id)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := h.liveStore.RememberChatLiveSession(ctx, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: native, Attention: attention, RecordedAt: h.clock()}); err != nil {
log.Printf("chat recovery journal session=%s failed (%T)", id, err)
return
}
if h.liveJournal == nil {
h.liveJournal = map[string]chatRecoveryState{}
}
h.liveJournal[id] = next
}
func (h *chatHub) touchRecoveryLocked(id string) {
if h.recoveryGeneration == nil {
h.recoveryGeneration = map[string]uint64{}
}
h.recoveryGeneration[id]++
}
// invalidateRecovery makes in-flight recovery probes stale before an explicit
// release. The durable record is retained until the native release succeeds,
// so a failed release remains recoverable.
func (h *chatHub) invalidateRecovery(id string) {
h.mu.Lock()
h.touchRecoveryLocked(id)
h.mu.Unlock()
}
// clearRecovery removes the durable liveness record after an explicit native
// release succeeds. The native transcript is untouched.
func (h *chatHub) clearRecovery(ctx context.Context, id string) error {
h.invalidateRecovery(id)
h.mu.Lock()
delete(h.liveJournal, id)
h.mu.Unlock()
if h.liveStore == nil {
return nil
}
return h.liveStore.ForgetChatLiveSession(ctx, id)
}
// Probe results are older than any lifecycle that arrived while they were in
// flight. Hold the same lock as lifecycle journaling through the final delete.
func (h *chatHub) forgetLive(ctx context.Context, id string, generation uint64) {
if h.liveStore == nil {
return
}
h.mu.Lock()
defer h.mu.Unlock()
if h.recoveryGeneration[id] != generation || h.rooms[id] != nil || ctx.Err() != nil {
return
}
if err := h.liveStore.ForgetChatLiveSession(ctx, id); err != nil {
log.Printf("chat recovery clear session=%s failed (%T)", id, err)
return
}
delete(h.liveJournal, id)
}
func (h *chatHub) rememberRecovered(ctx context.Context, v pipeline.ChatLiveSession, generation uint64) bool {
h.mu.Lock()
defer h.mu.Unlock()
if h.recoveryGeneration[v.SessionID] != generation || h.rooms[v.SessionID] != nil || ctx.Err() != nil {
return false
}
h.touchRecoveryLocked(v.SessionID)
if h.liveStore != nil {
if err := h.liveStore.RememberChatLiveSession(ctx, v); err != nil {
log.Printf("chat reattach journal session=%s failed (%T)", v.SessionID, err)
}
}
return true
}
// probeChatAttention reads only authoritative pending-work surfaces, never
// starts a model turn or infers task completion from absence of foreground work.
func probeChatAttention(ctx context.Context, broker chatBroker, id string) (attention, known bool, shells []protocol.ChatShell) {
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
shells, err := broker.Shells(probe, id)
if err != nil {
return false, false, nil
}
for _, shell := range shells {
if shell.Status == protocol.ChatShellRunning {
return true, true, shells
}
}
permissions, err := broker.Permissions(probe, id)
if err != nil {
return false, false, shells
}
for _, p := range permissions {
if !p.Resolved {
return true, true, shells
}
}
if reader, ok := broker.(chatFormReader); ok {
forms, err := reader.Forms(probe, id)
if err != nil {
return false, false, shells
}
for _, f := range forms {
if !f.Resolved && !f.Cancelled {
return true, true, shells
}
}
}
return false, true, shells
}
const chatRecoveryRetry = 30 * time.Second
// Only explicitly journaled but unobserved sessions are retried. Transient
// harness unavailability must survive more than one start, without polling all
// historical chats repeatedly.
func (h *chatHub) retryReattachments(ctx context.Context, broker chatBroker) {
ticker := time.NewTicker(chatRecoveryRetry)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
h.reattachSessions(ctx, broker)
}
}
}
// recoverDiscoveredSession attaches only if the harness reports actual work.
// Successful foreground-idle alone never rules out background tasks/questions.
func (h *chatHub) recoverDiscoveredSession(ctx context.Context, broker chatBroker, id string) {
h.mu.Lock()
observed := h.rooms[id] != nil
generation := h.recoveryGeneration[id]
h.mu.Unlock()
if observed || ctx.Err() != nil {
return
}
running := false
if prober, ok := broker.(chatTurnProber); ok {
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
busy, err := prober.ProbeTurn(probe, id)
cancel()
if err == nil {
running = busy
}
}
attention, known, shells := probeChatAttention(ctx, broker, id)
h.mu.Lock()
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
if !stale && known {
h.replaceShellsLocked(id, shells)
}
h.mu.Unlock()
if stale {
return
}
if !running && !attention {
return
}
session := pipeline.ChatLiveSession{SessionID: id, Turn: running, Attention: attention, RecordedAt: h.clock()}
if !h.rememberRecovered(ctx, session, generation) {
return
}
h.reattachSession(ctx, broker, session)
}