2307d45417
Ensure /new and chat close stop native foreground and background work before changing registry state. Preserve transcripts while surfacing release and recovery cleanup failures instead of reporting false success, with focused adapter and server regressions.
210 lines
6.2 KiB
Go
210 lines
6.2 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
type chatRecoveryState struct {
|
|
turn, attention bool
|
|
native string
|
|
}
|
|
|
|
// rememberLiveLocked journals liveness before an observer can be evicted or
|
|
// the process can crash. A bounded replay cache is never the recovery journal.
|
|
// Callers hold h.mu; like durable prompt admission, this small SQLite write is
|
|
// ordered with the lifecycle change it describes.
|
|
func (h *chatHub) rememberLiveLocked(id string) {
|
|
if h.liveStore == nil || (h.baseCtx != nil && h.baseCtx.Err() != nil) {
|
|
return
|
|
}
|
|
room := h.rooms[id]
|
|
if room == nil {
|
|
return
|
|
} // Losing observation is not authoritative idle.
|
|
turn := room.running || room.observedTurn
|
|
native := room.nativeTurnID
|
|
if current := h.turns[id]; current != nil && current.step != nil && current.step.Status == protocol.ChatStepStarted && !current.step.Intermediate {
|
|
turn = true
|
|
if native == "" {
|
|
native = current.step.TurnID
|
|
}
|
|
}
|
|
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
|
|
next := chatRecoveryState{turn: turn, attention: attention, native: native}
|
|
if previous, ok := h.liveJournal[id]; ok && previous == next {
|
|
return
|
|
}
|
|
h.touchRecoveryLocked(id)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
if err := h.liveStore.RememberChatLiveSession(ctx, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: native, Attention: attention, RecordedAt: h.clock()}); err != nil {
|
|
log.Printf("chat recovery journal session=%s failed (%T)", id, err)
|
|
return
|
|
}
|
|
if h.liveJournal == nil {
|
|
h.liveJournal = map[string]chatRecoveryState{}
|
|
}
|
|
h.liveJournal[id] = next
|
|
}
|
|
|
|
func (h *chatHub) touchRecoveryLocked(id string) {
|
|
if h.recoveryGeneration == nil {
|
|
h.recoveryGeneration = map[string]uint64{}
|
|
}
|
|
h.recoveryGeneration[id]++
|
|
}
|
|
|
|
// invalidateRecovery makes in-flight recovery probes stale before an explicit
|
|
// release. The durable record is retained until the native release succeeds,
|
|
// so a failed release remains recoverable.
|
|
func (h *chatHub) invalidateRecovery(id string) {
|
|
h.mu.Lock()
|
|
h.touchRecoveryLocked(id)
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
// clearRecovery removes the durable liveness record after an explicit native
|
|
// release succeeds. The native transcript is untouched.
|
|
func (h *chatHub) clearRecovery(ctx context.Context, id string) error {
|
|
h.invalidateRecovery(id)
|
|
h.mu.Lock()
|
|
delete(h.liveJournal, id)
|
|
h.mu.Unlock()
|
|
if h.liveStore == nil {
|
|
return nil
|
|
}
|
|
return h.liveStore.ForgetChatLiveSession(ctx, id)
|
|
}
|
|
|
|
// Probe results are older than any lifecycle that arrived while they were in
|
|
// flight. Hold the same lock as lifecycle journaling through the final delete.
|
|
func (h *chatHub) forgetLive(ctx context.Context, id string, generation uint64) {
|
|
if h.liveStore == nil {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if h.recoveryGeneration[id] != generation || h.rooms[id] != nil || ctx.Err() != nil {
|
|
return
|
|
}
|
|
if err := h.liveStore.ForgetChatLiveSession(ctx, id); err != nil {
|
|
log.Printf("chat recovery clear session=%s failed (%T)", id, err)
|
|
return
|
|
}
|
|
delete(h.liveJournal, id)
|
|
}
|
|
|
|
func (h *chatHub) rememberRecovered(ctx context.Context, v pipeline.ChatLiveSession, generation uint64) bool {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if h.recoveryGeneration[v.SessionID] != generation || h.rooms[v.SessionID] != nil || ctx.Err() != nil {
|
|
return false
|
|
}
|
|
h.touchRecoveryLocked(v.SessionID)
|
|
if h.liveStore != nil {
|
|
if err := h.liveStore.RememberChatLiveSession(ctx, v); err != nil {
|
|
log.Printf("chat reattach journal session=%s failed (%T)", v.SessionID, err)
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// probeChatAttention reads only authoritative pending-work surfaces, never
|
|
// starts a model turn or infers task completion from absence of foreground work.
|
|
func probeChatAttention(ctx context.Context, broker chatBroker, id string) (attention, known bool, shells []protocol.ChatShell) {
|
|
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
shells, err := broker.Shells(probe, id)
|
|
if err != nil {
|
|
return false, false, nil
|
|
}
|
|
for _, shell := range shells {
|
|
if shell.Status == protocol.ChatShellRunning {
|
|
return true, true, shells
|
|
}
|
|
}
|
|
permissions, err := broker.Permissions(probe, id)
|
|
if err != nil {
|
|
return false, false, shells
|
|
}
|
|
for _, p := range permissions {
|
|
if !p.Resolved {
|
|
return true, true, shells
|
|
}
|
|
}
|
|
if reader, ok := broker.(chatFormReader); ok {
|
|
forms, err := reader.Forms(probe, id)
|
|
if err != nil {
|
|
return false, false, shells
|
|
}
|
|
for _, f := range forms {
|
|
if !f.Resolved && !f.Cancelled {
|
|
return true, true, shells
|
|
}
|
|
}
|
|
}
|
|
return false, true, shells
|
|
}
|
|
|
|
const chatRecoveryRetry = 30 * time.Second
|
|
|
|
// Only explicitly journaled but unobserved sessions are retried. Transient
|
|
// harness unavailability must survive more than one start, without polling all
|
|
// historical chats repeatedly.
|
|
func (h *chatHub) retryReattachments(ctx context.Context, broker chatBroker) {
|
|
ticker := time.NewTicker(chatRecoveryRetry)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
h.reattachSessions(ctx, broker)
|
|
}
|
|
}
|
|
}
|
|
|
|
// recoverDiscoveredSession attaches only if the harness reports actual work.
|
|
// Successful foreground-idle alone never rules out background tasks/questions.
|
|
func (h *chatHub) recoverDiscoveredSession(ctx context.Context, broker chatBroker, id string) {
|
|
h.mu.Lock()
|
|
observed := h.rooms[id] != nil
|
|
generation := h.recoveryGeneration[id]
|
|
h.mu.Unlock()
|
|
if observed || ctx.Err() != nil {
|
|
return
|
|
}
|
|
running := false
|
|
if prober, ok := broker.(chatTurnProber); ok {
|
|
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
busy, err := prober.ProbeTurn(probe, id)
|
|
cancel()
|
|
if err == nil {
|
|
running = busy
|
|
}
|
|
}
|
|
attention, known, shells := probeChatAttention(ctx, broker, id)
|
|
h.mu.Lock()
|
|
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
|
|
if !stale && known {
|
|
h.replaceShellsLocked(id, shells)
|
|
}
|
|
h.mu.Unlock()
|
|
if stale {
|
|
return
|
|
}
|
|
if !running && !attention {
|
|
return
|
|
}
|
|
session := pipeline.ChatLiveSession{SessionID: id, Turn: running, Attention: attention, RecordedAt: h.clock()}
|
|
if !h.rememberRecovered(ctx, session, generation) {
|
|
return
|
|
}
|
|
h.reattachSession(ctx, broker, session)
|
|
}
|