188 lines
5.6 KiB
Go
188 lines
5.6 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
type chatRecoveryState struct {
|
|
turn, attention bool
|
|
native string
|
|
}
|
|
|
|
// rememberLiveLocked journals liveness before an observer can be evicted or
|
|
// the process can crash. A bounded replay cache is never the recovery journal.
|
|
// Callers hold h.mu; like durable prompt admission, this small SQLite write is
|
|
// ordered with the lifecycle change it describes.
|
|
func (h *chatHub) rememberLiveLocked(id string) {
|
|
if h.liveStore == nil || (h.baseCtx != nil && h.baseCtx.Err() != nil) {
|
|
return
|
|
}
|
|
room := h.rooms[id]
|
|
if room == nil {
|
|
return
|
|
} // Losing observation is not authoritative idle.
|
|
turn := room.running || room.observedTurn
|
|
native := room.nativeTurnID
|
|
if current := h.turns[id]; current != nil && current.step != nil && current.step.Status == protocol.ChatStepStarted && !current.step.Intermediate {
|
|
turn = true
|
|
if native == "" {
|
|
native = current.step.TurnID
|
|
}
|
|
}
|
|
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
|
|
next := chatRecoveryState{turn: turn, attention: attention, native: native}
|
|
if previous, ok := h.liveJournal[id]; ok && previous == next {
|
|
return
|
|
}
|
|
h.touchRecoveryLocked(id)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
if err := h.liveStore.RememberChatLiveSession(ctx, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: native, Attention: attention, RecordedAt: h.clock()}); err != nil {
|
|
log.Printf("chat recovery journal session=%s failed (%T)", id, err)
|
|
return
|
|
}
|
|
if h.liveJournal == nil {
|
|
h.liveJournal = map[string]chatRecoveryState{}
|
|
}
|
|
h.liveJournal[id] = next
|
|
}
|
|
|
|
func (h *chatHub) touchRecoveryLocked(id string) {
|
|
if h.recoveryGeneration == nil {
|
|
h.recoveryGeneration = map[string]uint64{}
|
|
}
|
|
h.recoveryGeneration[id]++
|
|
}
|
|
|
|
// Probe results are older than any lifecycle that arrived while they were in
|
|
// flight. Hold the same lock as lifecycle journaling through the final delete.
|
|
func (h *chatHub) forgetLive(ctx context.Context, id string, generation uint64) {
|
|
if h.liveStore == nil {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if h.recoveryGeneration[id] != generation || h.rooms[id] != nil || ctx.Err() != nil {
|
|
return
|
|
}
|
|
if err := h.liveStore.ForgetChatLiveSession(ctx, id); err != nil {
|
|
log.Printf("chat recovery clear session=%s failed (%T)", id, err)
|
|
return
|
|
}
|
|
delete(h.liveJournal, id)
|
|
}
|
|
|
|
func (h *chatHub) rememberRecovered(ctx context.Context, v pipeline.ChatLiveSession, generation uint64) bool {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if h.recoveryGeneration[v.SessionID] != generation || h.rooms[v.SessionID] != nil || ctx.Err() != nil {
|
|
return false
|
|
}
|
|
h.touchRecoveryLocked(v.SessionID)
|
|
if h.liveStore != nil {
|
|
if err := h.liveStore.RememberChatLiveSession(ctx, v); err != nil {
|
|
log.Printf("chat reattach journal session=%s failed (%T)", v.SessionID, err)
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// probeChatAttention reads only authoritative pending-work surfaces, never
|
|
// starts a model turn or infers task completion from absence of foreground work.
|
|
func probeChatAttention(ctx context.Context, broker chatBroker, id string) (attention, known bool, shells []protocol.ChatShell) {
|
|
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
shells, err := broker.Shells(probe, id)
|
|
if err != nil {
|
|
return false, false, nil
|
|
}
|
|
for _, shell := range shells {
|
|
if shell.Status == protocol.ChatShellRunning {
|
|
return true, true, shells
|
|
}
|
|
}
|
|
permissions, err := broker.Permissions(probe, id)
|
|
if err != nil {
|
|
return false, false, shells
|
|
}
|
|
for _, p := range permissions {
|
|
if !p.Resolved {
|
|
return true, true, shells
|
|
}
|
|
}
|
|
if reader, ok := broker.(chatFormReader); ok {
|
|
forms, err := reader.Forms(probe, id)
|
|
if err != nil {
|
|
return false, false, shells
|
|
}
|
|
for _, f := range forms {
|
|
if !f.Resolved && !f.Cancelled {
|
|
return true, true, shells
|
|
}
|
|
}
|
|
}
|
|
return false, true, shells
|
|
}
|
|
|
|
const chatRecoveryRetry = 30 * time.Second
|
|
|
|
// Only explicitly journaled but unobserved sessions are retried. Transient
|
|
// harness unavailability must survive more than one start, without polling all
|
|
// historical chats repeatedly.
|
|
func (h *chatHub) retryReattachments(ctx context.Context, broker chatBroker) {
|
|
ticker := time.NewTicker(chatRecoveryRetry)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
h.reattachSessions(ctx, broker)
|
|
}
|
|
}
|
|
}
|
|
|
|
// recoverDiscoveredSession attaches only if the harness reports actual work.
|
|
// Successful foreground-idle alone never rules out background tasks/questions.
|
|
func (h *chatHub) recoverDiscoveredSession(ctx context.Context, broker chatBroker, id string) {
|
|
h.mu.Lock()
|
|
observed := h.rooms[id] != nil
|
|
generation := h.recoveryGeneration[id]
|
|
h.mu.Unlock()
|
|
if observed || ctx.Err() != nil {
|
|
return
|
|
}
|
|
running := false
|
|
if prober, ok := broker.(chatTurnProber); ok {
|
|
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
busy, err := prober.ProbeTurn(probe, id)
|
|
cancel()
|
|
if err == nil {
|
|
running = busy
|
|
}
|
|
}
|
|
attention, known, shells := probeChatAttention(ctx, broker, id)
|
|
h.mu.Lock()
|
|
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
|
|
if !stale && known {
|
|
h.replaceShellsLocked(id, shells)
|
|
}
|
|
h.mu.Unlock()
|
|
if stale {
|
|
return
|
|
}
|
|
if !running && !attention {
|
|
return
|
|
}
|
|
session := pipeline.ChatLiveSession{SessionID: id, Turn: running, Attention: attention, RecordedAt: h.clock()}
|
|
if !h.rememberRecovered(ctx, session, generation) {
|
|
return
|
|
}
|
|
h.reattachSession(ctx, broker, session)
|
|
}
|