Files
pipelines/internal/server/chat_recovery_journal.go
T

188 lines
5.6 KiB
Go

package server
import (
"context"
"log"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type chatRecoveryState struct {
turn, attention bool
native string
}
// rememberLiveLocked journals liveness before an observer can be evicted or
// the process can crash. A bounded replay cache is never the recovery journal.
// Callers hold h.mu; like durable prompt admission, this small SQLite write is
// ordered with the lifecycle change it describes.
func (h *chatHub) rememberLiveLocked(id string) {
if h.liveStore == nil || (h.baseCtx != nil && h.baseCtx.Err() != nil) {
return
}
room := h.rooms[id]
if room == nil {
return
} // Losing observation is not authoritative idle.
turn := room.running || room.observedTurn
native := room.nativeTurnID
if current := h.turns[id]; current != nil && current.step != nil && current.step.Status == protocol.ChatStepStarted && !current.step.Intermediate {
turn = true
if native == "" {
native = current.step.TurnID
}
}
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
next := chatRecoveryState{turn: turn, attention: attention, native: native}
if previous, ok := h.liveJournal[id]; ok && previous == next {
return
}
h.touchRecoveryLocked(id)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := h.liveStore.RememberChatLiveSession(ctx, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: native, Attention: attention, RecordedAt: h.clock()}); err != nil {
log.Printf("chat recovery journal session=%s failed (%T)", id, err)
return
}
if h.liveJournal == nil {
h.liveJournal = map[string]chatRecoveryState{}
}
h.liveJournal[id] = next
}
func (h *chatHub) touchRecoveryLocked(id string) {
if h.recoveryGeneration == nil {
h.recoveryGeneration = map[string]uint64{}
}
h.recoveryGeneration[id]++
}
// Probe results are older than any lifecycle that arrived while they were in
// flight. Hold the same lock as lifecycle journaling through the final delete.
func (h *chatHub) forgetLive(ctx context.Context, id string, generation uint64) {
if h.liveStore == nil {
return
}
h.mu.Lock()
defer h.mu.Unlock()
if h.recoveryGeneration[id] != generation || h.rooms[id] != nil || ctx.Err() != nil {
return
}
if err := h.liveStore.ForgetChatLiveSession(ctx, id); err != nil {
log.Printf("chat recovery clear session=%s failed (%T)", id, err)
return
}
delete(h.liveJournal, id)
}
func (h *chatHub) rememberRecovered(ctx context.Context, v pipeline.ChatLiveSession, generation uint64) bool {
h.mu.Lock()
defer h.mu.Unlock()
if h.recoveryGeneration[v.SessionID] != generation || h.rooms[v.SessionID] != nil || ctx.Err() != nil {
return false
}
h.touchRecoveryLocked(v.SessionID)
if h.liveStore != nil {
if err := h.liveStore.RememberChatLiveSession(ctx, v); err != nil {
log.Printf("chat reattach journal session=%s failed (%T)", v.SessionID, err)
}
}
return true
}
// probeChatAttention reads only authoritative pending-work surfaces, never
// starts a model turn or infers task completion from absence of foreground work.
func probeChatAttention(ctx context.Context, broker chatBroker, id string) (attention, known bool, shells []protocol.ChatShell) {
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
shells, err := broker.Shells(probe, id)
if err != nil {
return false, false, nil
}
for _, shell := range shells {
if shell.Status == protocol.ChatShellRunning {
return true, true, shells
}
}
permissions, err := broker.Permissions(probe, id)
if err != nil {
return false, false, shells
}
for _, p := range permissions {
if !p.Resolved {
return true, true, shells
}
}
if reader, ok := broker.(chatFormReader); ok {
forms, err := reader.Forms(probe, id)
if err != nil {
return false, false, shells
}
for _, f := range forms {
if !f.Resolved && !f.Cancelled {
return true, true, shells
}
}
}
return false, true, shells
}
const chatRecoveryRetry = 30 * time.Second
// Only explicitly journaled but unobserved sessions are retried. Transient
// harness unavailability must survive more than one start, without polling all
// historical chats repeatedly.
func (h *chatHub) retryReattachments(ctx context.Context, broker chatBroker) {
ticker := time.NewTicker(chatRecoveryRetry)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
h.reattachSessions(ctx, broker)
}
}
}
// recoverDiscoveredSession attaches only if the harness reports actual work.
// Successful foreground-idle alone never rules out background tasks/questions.
func (h *chatHub) recoverDiscoveredSession(ctx context.Context, broker chatBroker, id string) {
h.mu.Lock()
observed := h.rooms[id] != nil
generation := h.recoveryGeneration[id]
h.mu.Unlock()
if observed || ctx.Err() != nil {
return
}
running := false
if prober, ok := broker.(chatTurnProber); ok {
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
busy, err := prober.ProbeTurn(probe, id)
cancel()
if err == nil {
running = busy
}
}
attention, known, shells := probeChatAttention(ctx, broker, id)
h.mu.Lock()
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
if !stale && known {
h.replaceShellsLocked(id, shells)
}
h.mu.Unlock()
if stale {
return
}
if !running && !attention {
return
}
session := pipeline.ChatLiveSession{SessionID: id, Turn: running, Attention: attention, RecordedAt: h.clock()}
if !h.rememberRecovered(ctx, session, generation) {
return
}
h.reattachSession(ctx, broker, session)
}