diff --git a/internal/pipeline/chat_live_sessions.go b/internal/pipeline/chat_live_sessions.go index ae6bb973..03e52e2f 100644 --- a/internal/pipeline/chat_live_sessions.go +++ b/internal/pipeline/chat_live_sessions.go @@ -88,3 +88,56 @@ func (s *Store) TakeChatLiveSessions(ctx context.Context) ([]ChatLiveSession, er } return sessions, tx.Commit() } + +// ChatLiveSessions reads recovery membership without consuming it. A failed +// reconnect or a second crash must not erase the only record of live work. +func (s *Store) ChatLiveSessions(ctx context.Context) ([]ChatLiveSession, error) { + rows, err := s.db.QueryContext(ctx, `SELECT session_id,backend,turn,native_turn_id,attention,recorded_at FROM chat_live_sessions ORDER BY recorded_at, session_id`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []ChatLiveSession + for rows.Next() { + var v ChatLiveSession + var at int64 + if err := rows.Scan(&v.SessionID, &v.Backend, &v.Turn, &v.NativeTurnID, &v.Attention, &at); err != nil { + return nil, err + } + v.RecordedAt = time.UnixMilli(at) + out = append(out, v) + } + return out, rows.Err() +} + +// RememberChatLiveSession journals live work as it is observed, not only at a +// graceful shutdown. Attention remains sticky until an authoritative reconnect +// confirms idle: evicting a bounded replay cache is not evidence tasks ended. +func (s *Store) RememberChatLiveSession(ctx context.Context, v ChatLiveSession) error { + if v.RecordedAt.IsZero() { + v.RecordedAt = time.Now() + } + if !v.Turn && !v.Attention { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `UPDATE chat_live_sessions SET turn=0,native_turn_id='' WHERE session_id=?`, v.SessionID); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions WHERE session_id=? AND turn=0 AND attention=0`, v.SessionID); err != nil { + return err + } + return tx.Commit() + } + + _, err := s.db.ExecContext(ctx, `INSERT INTO chat_live_sessions(session_id,backend,turn,native_turn_id,attention,recorded_at) VALUES(?,?,?,?,?,?) ON CONFLICT(session_id) DO UPDATE SET backend=excluded.backend,turn=excluded.turn,native_turn_id=CASE WHEN excluded.turn=0 THEN '' WHEN excluded.native_turn_id!='' THEN excluded.native_turn_id ELSE chat_live_sessions.native_turn_id END,attention=MAX(chat_live_sessions.attention,excluded.attention),recorded_at=excluded.recorded_at`, v.SessionID, v.Backend, v.Turn, v.NativeTurnID, v.Attention, v.RecordedAt.UnixMilli()) + return err +} + +// ForgetChatLiveSession is used only after authoritative idle or explicit close. +func (s *Store) ForgetChatLiveSession(ctx context.Context, id string) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM chat_live_sessions WHERE session_id=?`, id) + return err +} diff --git a/internal/pipeline/chat_live_sessions_test.go b/internal/pipeline/chat_live_sessions_test.go new file mode 100644 index 00000000..806f862d --- /dev/null +++ b/internal/pipeline/chat_live_sessions_test.go @@ -0,0 +1,36 @@ +package pipeline + +import ( + "context" + "path/filepath" + "testing" +) + +func TestChatRecoveryJournalIsNotConsumedAndKeepsUnverifiedAttention(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "live.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + ctx := context.Background() + if err := s.RememberChatLiveSession(ctx, ChatLiveSession{SessionID: "ses_task", Attention: true}); err != nil { + t.Fatal(err) + } + // Ending the foreground turn/evicting its replay is not task completion. + if err := s.RememberChatLiveSession(ctx, ChatLiveSession{SessionID: "ses_task"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + got, err := s.ChatLiveSessions(ctx) + if err != nil || len(got) != 1 || !got[0].Attention || got[0].Turn { + t.Fatalf("read%d: %+v %v", i, got, err) + } + } + if err := s.ForgetChatLiveSession(ctx, "ses_task"); err != nil { + t.Fatal(err) + } + got, err := s.ChatLiveSessions(ctx) + if err != nil || len(got) != 0 { + t.Fatalf("confirmed idle retained: %+v %v", got, err) + } +} diff --git a/internal/server/chat_activity.go b/internal/server/chat_activity.go index 8e769949..098dd48d 100644 --- a/internal/server/chat_activity.go +++ b/internal/server/chat_activity.go @@ -53,6 +53,7 @@ func (h *chatHub) noteActivityLocked(sessionID string) { if sessionID == "" { return } + h.rememberLiveLocked(sessionID) next := h.chatActivityLocked(sessionID) previous := h.activity[sessionID] if previous.TurnRunning == next.TurnRunning && previous.BackgroundTasks == next.BackgroundTasks { diff --git a/internal/server/chat_activity_test.go b/internal/server/chat_activity_test.go index 0fd72740..8ee57b3a 100644 --- a/internal/server/chat_activity_test.go +++ b/internal/server/chat_activity_test.go @@ -109,10 +109,9 @@ func TestChatActivityReachesEveryConnection(t *testing.T) { } } -// A background task that never reports its exit does not hold the observer -// forever: it is released after chatUnattendedTurnLimit, and the list then -// reports no tasks rather than a stale count. -func TestChatUnreportedBackgroundTaskIsReleased(t *testing.T) { +// A running task is not an idle observer: the old observer TTL must not +// invent completion. A terminal event releases it and clears list activity. +func TestChatBackgroundTaskWaitsForExitBeyondObserverTTL(t *testing.T) { fastUnattended(t) t.Setenv("PIPELINES_NTFY_URL", "") limit := chatUnattendedTurnLimit @@ -134,17 +133,21 @@ func TestChatUnreportedBackgroundTaskIsReleased(t *testing.T) { broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1")) readActivity(t, server.ctx, list) viewer.CloseNow() - time.Sleep(100 * time.Millisecond) + time.Sleep(2 * chatUnattendedTurnLimit) if active, _ := broker.streams(); active != 1 { - t.Fatalf("background task not observed: %d", active) + t.Fatalf("background observer expired: %d", active) } + if got := listActivity(t, server.ctx, list); got == nil || got.BackgroundTasks != 1 { + t.Fatalf("invented task completion: %+v", got) + } + broker.push(shellFrame("sh_1", protocol.ChatShellExited)) released := readActivity(t, server.ctx, list) if released.TurnRunning || released.BackgroundTasks != 0 { t.Fatalf("release push = %+v", released) } waitFor(t, "observer released", func() bool { active, _ := broker.streams(); return active == 0 }) if got := listActivity(t, server.ctx, list); got == nil || got.BackgroundTasks != 0 { - t.Fatalf("listed activity after expiry = %+v", got) + t.Fatalf("listed activity after completion = %+v", got) } } diff --git a/internal/server/chat_background_recovery_test.go b/internal/server/chat_background_recovery_test.go new file mode 100644 index 00000000..972214e2 --- /dev/null +++ b/internal/server/chat_background_recovery_test.go @@ -0,0 +1,246 @@ +package server + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type recoveryTasks struct { + *restartChat + mu sync.Mutex + tasks map[string][]protocol.ChatShell + unavailable bool + historyReads int + creates int +} + +func newRecoveryTasks() *recoveryTasks { + return &recoveryTasks{restartChat: newRestartChat(), tasks: map[string][]protocol.ChatShell{}} +} +func (b *recoveryTasks) Shells(_ context.Context, id string) ([]protocol.ChatShell, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.unavailable { + return nil, errors.New("temporary read failure") + } + return append([]protocol.ChatShell(nil), b.tasks[id]...), nil +} +func (b *recoveryTasks) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.historyReads++ + return protocol.ChatHistoryPage{}, nil +} +func (b *recoveryTasks) Create(context.Context, string, string) (protocol.ChatSession, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.creates++ + return protocol.ChatSession{}, errors.New("recovery must not create a session") +} +func task(id string) protocol.ChatShell { + return protocol.ChatShell{ID: id, Status: protocol.ChatShellRunning, Command: "isolated background task", StartedAt: 1234} +} +func waitRecoveredTask(t *testing.T, s *chatRestartServer, id, taskID string) { + t.Helper() + waitFor(t, "background reattached with same identity", func() bool { + h := s.handler.hub + h.mu.Lock() + room := h.rooms[id] + attached := room != nil && room.streaming + h.mu.Unlock() + if !attached { + return false + } + got := h.shellSnapshot(id) + return len(got) == 1 && got[0].ID == taskID && got[0].StartedAt == 1234 + }) +} + +func TestBackgroundRecoverySurvivesRoomEvictionAndUngracefulStop(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + first := newRecoveryTasks() + first.tasks["ses_1"] = []protocol.ChatShell{task("job_same")} + s := startChatRestartServer(t, first, path) + waitRecoveredTask(t, s, "ses_1", "job_same") + h := s.handler.hub + h.mu.Lock() + h.releaseRoomLocked(h.rooms["ses_1"]) + h.mu.Unlock() + // No DetachSessions: model a crash after the observer was released. + s.stop() + second := newRecoveryTasks() + second.tasks["ses_1"] = []protocol.ChatShell{task("job_same")} + next := startChatRestartServer(t, second, path) + waitRecoveredTask(t, next, "ses_1", "job_same") + second.mu.Lock() + defer second.mu.Unlock() + if second.historyReads != 0 || second.creates != 0 || second.promptCount() != 0 { + t.Fatal("recovery started/read a conversation instead of reattaching tasks") + } +} + +func TestBackgroundRecoveryRetainsJournalAcrossFailedReattachment(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + store := openQueueStore(t, path) + if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}); err != nil { + t.Fatal(err) + } + broken := newRecoveryTasks() + broken.unavailable = true + first := startChatRestartServer(t, broken, path) + select { + case <-first.handler.hub.recoveryDone: + case <-first.ctx.Done(): + t.Fatal("recovery did not settle") + } + records, err := store.ChatLiveSessions(context.Background()) + if err != nil || len(records) != 1 { + t.Fatalf("failed reconnect consumed journal: %+v %v", records, err) + } + first.stop() + repaired := newRecoveryTasks() + repaired.tasks["ses_1"] = []protocol.ChatShell{task("job_kept")} + next := startChatRestartServer(t, repaired, path) + waitRecoveredTask(t, next, "ses_1", "job_kept") +} + +func TestBackgroundRecoveryFindsOldSessionsBeyondFormerRecentLimit(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + store := openQueueStore(t, path) + for i := 0; i < chatRecoverSessionLimit+5; i++ { + if err := store.RecordOperatorSession(context.Background(), fmt.Sprintf("ses_new_%02d", i), t.TempDir(), "idle", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`UPDATE operator_sessions SET last_used_at=? WHERE session_id='ses_1'`, time.Now().Add(-7*24*time.Hour).UnixMilli()); err != nil { + t.Fatal(err) + } + broker := newRecoveryTasks() + broker.tasks["ses_1"] = []protocol.ChatShell{task("job_old")} + s := startChatRestartServer(t, broker, path) + waitRecoveredTask(t, s, "ses_1", "job_old") + broker.mu.Lock() + defer broker.mu.Unlock() + if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 { + t.Fatal("discovery resumed or read chat history") + } +} + +func TestBackgroundRecoveryKeepsLiveObserversBeyondIdleCapAndTTL(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + store := openQueueStore(t, path) + broker := newRecoveryTasks() + count := chatUnattendedLimit + 3 + for i := 0; i < count; i++ { + id := fmt.Sprintf("ses_busy_%02d", i) + if err := store.RecordOperatorSession(context.Background(), id, t.TempDir(), "background", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + broker.tasks[id] = []protocol.ChatShell{task("job_" + id)} + } + s := startChatRestartServer(t, broker, path) + waitFor(t, "all background streams restored despite idle cap", func() bool { active, _ := broker.streams(); return active == count }) + h := s.handler.hub + h.mu.Lock() + rooms := make([]*chatRoom, 0, len(h.rooms)) + for _, room := range h.rooms { + room.unattendedSince = h.clock().Add(-7 * time.Hour) + rooms = append(rooms, room) + } + h.mu.Unlock() + for _, room := range rooms { + if h.reapUnattended(room) { + t.Fatal("live task observer expired") + } + } + if active, _ := broker.streams(); active != count { + t.Fatalf("lost background streams: %d", active) + } +} + +// A readback captured before a newer task event must not erase that task's +// durable recovery membership, even if its observer has since been released. +type staleRecoveryTasks struct { + *recoveryTasks + entered chan struct{} + release chan struct{} +} + +func (b *staleRecoveryTasks) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) { + close(b.entered) + select { + case <-b.release: + return nil, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} +func TestBackgroundRecoveryStaleIdleCannotForgetNewTask(t *testing.T) { + ctx := context.Background() + store := openQueueStore(t, restartChatDB(t)) + old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true} + if err := store.RememberChatLiveSession(ctx, old); err != nil { + t.Fatal(err) + } + h := newChatHub() + h.liveStore = store + broker := &staleRecoveryTasks{recoveryTasks: newRecoveryTasks(), entered: make(chan struct{}), release: make(chan struct{})} + done := make(chan struct{}) + go func() { h.reattachSession(ctx, broker, old); close(done) }() + <-broker.entered + h.mu.Lock() + room := h.roomLocked("ses_1") + newest := task("new_job") + h.applyLocked(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &newest}) + h.releaseRoomLocked(room) + h.mu.Unlock() + close(broker.release) + <-done + rows, err := store.ChatLiveSessions(ctx) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || !rows[0].Attention { + t.Fatalf("stale idle erased new task journal: %+v", rows) + } +} + +func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T) { + ctx := context.Background() + store := openQueueStore(t, restartChatDB(t)) + old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true} + if err := store.RememberChatLiveSession(ctx, old); err != nil { + t.Fatal(err) + } + h := newChatHub() + h.liveStore = store + broker := newRecoveryTasks() + h.reattachSession(ctx, broker, old) + rows, err := store.ChatLiveSessions(ctx) + if err != nil { + t.Fatal(err) + } + if len(rows) != 0 { + t.Fatalf("authoritative idle left stale journal: %+v", rows) + } + if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 { + t.Fatal("idle recovery started or read a conversation") + } +} diff --git a/internal/server/chat_detach.go b/internal/server/chat_detach.go index 6b57858d..4849b774 100644 --- a/internal/server/chat_detach.go +++ b/internal/server/chat_detach.go @@ -28,11 +28,12 @@ import ( // OpenCode (an attached or detached owned service), Codex (its App Server) and // Claude Code (its detachable hosts) implement Detach. -// chatLiveStore records, on shutdown, the chats left live on their harness so -// the next start reattaches exactly those; *pipeline.Store implements it. +// chatLiveStore durably journals live work before room eviction/crash, as well +// as on graceful shutdown; *pipeline.Store implements it. type chatLiveStore interface { - RecordChatLiveSessions(ctx context.Context, sessions []pipeline.ChatLiveSession) error - TakeChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error) + ChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error) + RememberChatLiveSession(ctx context.Context, session pipeline.ChatLiveSession) error + ForgetChatLiveSession(ctx context.Context, id string) error } // chatReattachPoll is how often a reattached turn re-asks its harness whether @@ -63,10 +64,15 @@ func (h *chatHub) detachSessions(ctx context.Context) error { if len(live) > 0 { log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live)) } - return h.liveStore.RecordChatLiveSessions(ctx, live) + for _, session := range live { + if err := h.liveStore.RememberChatLiveSession(ctx, session); err != nil { + return err + } + } + return nil } -// reattachSessions is the start-up half of detachSessions: each recorded chat +// reattachSessions reads (without consuming) the recovery journal. Each chat // is observed again (stream, pending questions, background tasks) while its // harness still has work, and a turn that ended while the server was down is // folded in from the harness's history, so its reply still notifies and its @@ -75,7 +81,7 @@ func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) { if h.liveStore == nil || broker == nil { return } - live, err := h.liveStore.TakeChatLiveSessions(ctx) + live, err := h.liveStore.ChatLiveSessions(ctx) if err != nil { log.Printf("chat reattach: %v", err) return @@ -87,13 +93,34 @@ func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) { if !validChatSessionID(session.SessionID) { continue } + h.mu.Lock() + observed := h.rooms[session.SessionID] != nil + generation := h.recoveryGeneration[session.SessionID] + h.mu.Unlock() + if observed { + continue + } + if h.registry != nil { + row, found, err := h.registry.OperatorSession(ctx, session.SessionID) + if err != nil { + continue + } + if !found || row.ClosedAt != 0 { + h.forgetLive(ctx, session.SessionID, generation) + continue + } + } h.reattachSession(ctx, broker, session) } } func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) { id := session.SessionID + h.mu.Lock() + generation := h.recoveryGeneration[id] + h.mu.Unlock() running := false + turnKnown := false if prober, ok := broker.(chatTurnProber); ok { probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) busy, err := prober.ProbeTurn(probeCtx, id) @@ -104,6 +131,54 @@ func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, sessio busy = session.Turn } running = busy + turnKnown = err == nil + } + var snapshot []protocol.ChatShell + attentionKnown := false + if !running { + // A recovered task may have ended while disconnected. Only clear the + // journal after all pending-work surfaces authoritatively report idle. + if attention, known, shells := probeChatAttention(ctx, broker, id); known { + snapshot = shells + attentionKnown = true + session.Attention = attention + } else { + session.Attention = true // Unknown pending work is not idle. + } + } + h.mu.Lock() + stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil + if !stale && attentionKnown { + h.replaceShellsLocked(id, snapshot) + } + h.mu.Unlock() + if stale { + return + } + if turnKnown && !running && !session.Attention { + if session.Turn { + h.foldEndedTurn(ctx, broker, session, nil) + } + if h.liveStore != nil { + h.forgetLive(ctx, id, generation) + } + return + } + retained := session + retained.Turn = running + retained.RecordedAt = h.clock() + if !h.rememberRecovered(ctx, retained, generation) { + return + } + if running && session.Turn { + // Seed ownership before opening the stream: replay/poll admission can + // journal immediately and must not overwrite the recovered running turn. + h.mu.Lock() + turn := h.turnLocked(id) + if turn.step == nil || turn.step.Status == protocol.ChatStepEnded { + turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID} + } + h.mu.Unlock() } if running || session.Attention { h.observeUnattended(id, broker) diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 5a0e2f76..18b2a8d9 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -56,7 +56,10 @@ type chatHub struct { queueStore chatPromptQueueStore // liveStore records the chats a stopping server leaves live on their // harness (chat_detach.go); nil reattaches nothing beyond the start probe. - liveStore chatLiveStore + liveStore chatLiveStore + liveJournal map[string]chatRecoveryState + recoveryGeneration map[string]uint64 + recoveryDone <-chan struct{} // recentOps/recentOpOrder are the bounded record of accepted prompt // operation ids, so a client retry is acknowledged instead of re-sent. recentOps map[string]chatRecentOp @@ -202,7 +205,11 @@ func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) { if room := h.rooms[frame.SessionID]; room != nil { room.shellReadback.Record(frame.Shell) } + if frame.Shell != nil && frame.Shell.ID != "" { + h.touchRecoveryLocked(frame.SessionID) + } turn.applyShell(frame.Shell) + h.rememberLiveLocked(frame.SessionID) return case protocol.ChatEventForm: // A form/question likewise carries no message id. Keep the pending form @@ -221,6 +228,7 @@ func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) { if !frame.Form.Resolved && !frame.Form.Cancelled && len(room.forms) < 8 { room.forms = append(room.forms, *frame.Form) } + h.rememberLiveLocked(frame.SessionID) } return } diff --git a/internal/server/chat_recovery_journal.go b/internal/server/chat_recovery_journal.go new file mode 100644 index 00000000..b499d86b --- /dev/null +++ b/internal/server/chat_recovery_journal.go @@ -0,0 +1,187 @@ +package server + +import ( + "context" + "log" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type chatRecoveryState struct { + turn, attention bool + native string +} + +// rememberLiveLocked journals liveness before an observer can be evicted or +// the process can crash. A bounded replay cache is never the recovery journal. +// Callers hold h.mu; like durable prompt admission, this small SQLite write is +// ordered with the lifecycle change it describes. +func (h *chatHub) rememberLiveLocked(id string) { + if h.liveStore == nil || (h.baseCtx != nil && h.baseCtx.Err() != nil) { + return + } + room := h.rooms[id] + if room == nil { + return + } // Losing observation is not authoritative idle. + turn := room.running || room.observedTurn + native := room.nativeTurnID + if current := h.turns[id]; current != nil && current.step != nil && current.step.Status == protocol.ChatStepStarted && !current.step.Intermediate { + turn = true + if native == "" { + native = current.step.TurnID + } + } + attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0 + next := chatRecoveryState{turn: turn, attention: attention, native: native} + if previous, ok := h.liveJournal[id]; ok && previous == next { + return + } + h.touchRecoveryLocked(id) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := h.liveStore.RememberChatLiveSession(ctx, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: native, Attention: attention, RecordedAt: h.clock()}); err != nil { + log.Printf("chat recovery journal session=%s failed (%T)", id, err) + return + } + if h.liveJournal == nil { + h.liveJournal = map[string]chatRecoveryState{} + } + h.liveJournal[id] = next +} + +func (h *chatHub) touchRecoveryLocked(id string) { + if h.recoveryGeneration == nil { + h.recoveryGeneration = map[string]uint64{} + } + h.recoveryGeneration[id]++ +} + +// Probe results are older than any lifecycle that arrived while they were in +// flight. Hold the same lock as lifecycle journaling through the final delete. +func (h *chatHub) forgetLive(ctx context.Context, id string, generation uint64) { + if h.liveStore == nil { + return + } + h.mu.Lock() + defer h.mu.Unlock() + if h.recoveryGeneration[id] != generation || h.rooms[id] != nil || ctx.Err() != nil { + return + } + if err := h.liveStore.ForgetChatLiveSession(ctx, id); err != nil { + log.Printf("chat recovery clear session=%s failed (%T)", id, err) + return + } + delete(h.liveJournal, id) +} + +func (h *chatHub) rememberRecovered(ctx context.Context, v pipeline.ChatLiveSession, generation uint64) bool { + h.mu.Lock() + defer h.mu.Unlock() + if h.recoveryGeneration[v.SessionID] != generation || h.rooms[v.SessionID] != nil || ctx.Err() != nil { + return false + } + h.touchRecoveryLocked(v.SessionID) + if h.liveStore != nil { + if err := h.liveStore.RememberChatLiveSession(ctx, v); err != nil { + log.Printf("chat reattach journal session=%s failed (%T)", v.SessionID, err) + } + } + return true +} + +// probeChatAttention reads only authoritative pending-work surfaces, never +// starts a model turn or infers task completion from absence of foreground work. +func probeChatAttention(ctx context.Context, broker chatBroker, id string) (attention, known bool, shells []protocol.ChatShell) { + probe, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + shells, err := broker.Shells(probe, id) + if err != nil { + return false, false, nil + } + for _, shell := range shells { + if shell.Status == protocol.ChatShellRunning { + return true, true, shells + } + } + permissions, err := broker.Permissions(probe, id) + if err != nil { + return false, false, shells + } + for _, p := range permissions { + if !p.Resolved { + return true, true, shells + } + } + if reader, ok := broker.(chatFormReader); ok { + forms, err := reader.Forms(probe, id) + if err != nil { + return false, false, shells + } + for _, f := range forms { + if !f.Resolved && !f.Cancelled { + return true, true, shells + } + } + } + return false, true, shells +} + +const chatRecoveryRetry = 30 * time.Second + +// Only explicitly journaled but unobserved sessions are retried. Transient +// harness unavailability must survive more than one start, without polling all +// historical chats repeatedly. +func (h *chatHub) retryReattachments(ctx context.Context, broker chatBroker) { + ticker := time.NewTicker(chatRecoveryRetry) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + h.reattachSessions(ctx, broker) + } + } +} + +// recoverDiscoveredSession attaches only if the harness reports actual work. +// Successful foreground-idle alone never rules out background tasks/questions. +func (h *chatHub) recoverDiscoveredSession(ctx context.Context, broker chatBroker, id string) { + h.mu.Lock() + observed := h.rooms[id] != nil + generation := h.recoveryGeneration[id] + h.mu.Unlock() + if observed || ctx.Err() != nil { + return + } + running := false + if prober, ok := broker.(chatTurnProber); ok { + probe, cancel := context.WithTimeout(ctx, 5*time.Second) + busy, err := prober.ProbeTurn(probe, id) + cancel() + if err == nil { + running = busy + } + } + attention, known, shells := probeChatAttention(ctx, broker, id) + h.mu.Lock() + stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil + if !stale && known { + h.replaceShellsLocked(id, shells) + } + h.mu.Unlock() + if stale { + return + } + if !running && !attention { + return + } + session := pipeline.ChatLiveSession{SessionID: id, Turn: running, Attention: attention, RecordedAt: h.clock()} + if !h.rememberRecovered(ctx, session, generation) { + return + } + h.reattachSession(ctx, broker, session) +} diff --git a/internal/server/chat_room.go b/internal/server/chat_room.go index 99f94a8e..24ade018 100644 --- a/internal/server/chat_room.go +++ b/internal/server/chat_room.go @@ -343,6 +343,7 @@ func (h *chatHub) releaseRoomLocked(room *chatRoom) { if h.rooms[room.sessionID] != room { return } + h.rememberLiveLocked(room.sessionID) h.clearCompactionLocked(room) delete(h.rooms, room.sessionID) room.cancel() @@ -709,6 +710,7 @@ func (h *chatHub) pushPermission(sessionID string, frame protocol.ChatPermission } } room.pending = append(room.pending, frame) + h.rememberLiveLocked(sessionID) h.notePermissionAttentionLocked(frame) if len(room.pending) > chatRoomPendingLimit { room.pending = room.pending[len(room.pending)-chatRoomPendingLimit:] diff --git a/internal/server/chat_shell_output_test.go b/internal/server/chat_shell_output_test.go index 5961c26b..fba6ad82 100644 --- a/internal/server/chat_shell_output_test.go +++ b/internal/server/chat_shell_output_test.go @@ -85,7 +85,13 @@ func testChatShellOutputOwnedAndForeign(t *testing.T, exited bool) { if err := store.RecordOperatorSession(ctx, LargeChatFixtureSessionID, t.TempDir(), "owned", protocol.ChatSessionTagOperator); err != nil { t.Fatal(err) } - srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store})) + handler := HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store}).(*runHandler) + srv := httptest.NewServer(handler) + select { + case <-handler.hub.recoveryDone: + case <-ctx.Done(): + t.Fatal("startup recovery did not finish") + } defer srv.Close() ws := dialShellOutputWebsocket(t, ctx, srv) defer ws.CloseNow() diff --git a/internal/server/chat_unattended.go b/internal/server/chat_unattended.go index 05719cc1..34a1910a 100644 --- a/internal/server/chat_unattended.go +++ b/internal/server/chat_unattended.go @@ -3,6 +3,7 @@ package server import ( "context" "log" + "sync" "time" ) @@ -17,11 +18,10 @@ import ( // observation is started at server start for sessions whose harness reports a // running turn, so a reply that finishes after a restart still notifies. // -// Resources are bounded: at most chatUnattendedLimit rooms are kept without a -// client (rooms holding queued prompts are not counted: they were always kept -// until delivered), a stuck turn or a background task that never reports an -// exit is released after chatUnattendedTurnLimit, and an idle session is never -// observed. +// Idle/stale foreground observers are bounded by chatUnattendedLimit and +// chatUnattendedTurnLimit. Known running background work (like queued prompts) +// is not an idle lease: it remains attached until actual completion or explicit +// stop. Its durable journal is independent of bounded replay/room retention. // chatUnattendedLimit bounds the rooms kept observing with no client attached // and no queued prompt. @@ -73,10 +73,8 @@ func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool { if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 { return true } - // Running background tasks keep the stream observed so their exits are - // seen and the chat list's indicator clears. This is bounded like a turn: - // the unattended room limit, the chatUnattendedTurnLimit expiry, and the - // harness itself (a Claude Code process that exits ends its tasks). + // Running background tasks keep the stream observed so their actual exits + // are seen and the chat list's indicator clears, without a fixed age cap. if h.backgroundTasksLocked(room.sessionID) > 0 { return true } @@ -108,7 +106,7 @@ func (h *chatHub) leaveUnattendedLocked(room *chatRoom) { h.releaseRoomLocked(room) return } - if !room.watching && h.unattendedCountLocked(room) >= chatUnattendedLimit { + if !room.watching && h.backgroundTasksLocked(room.sessionID) == 0 && h.unattendedCountLocked(room) >= chatUnattendedLimit { log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit) h.releaseRoomLocked(room) return @@ -172,6 +170,11 @@ func (h *chatHub) reapUnattended(room *chatRoom) bool { room.unattendedSince = h.clock() return false } + // A running task is actual work, not an idle-observer lease. Its terminal + // lifecycle/readback, or explicit operator stop, releases it normally. + if h.backgroundTasksLocked(room.sessionID) > 0 { + return false + } expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit if !expired && h.keepUnattendedLocked(room, busy, true) { return false @@ -199,45 +202,56 @@ func (h *chatHub) attendedLocked(room *chatRoom) bool { return false } -// observeRunningTurns is the start-up half: each recently used open operator -// session whose harness reports a turn in flight gets an unattended room -// observing its stream, so the reply (or a question) still notifies. It is -// best effort per harness: OpenCode and Codex keep running across a Pipelines -// restart and report it; Claude Code runs as a child of the server and so has -// no turn left after a restart. +// observeRunningTurns discovers live foreground or background work even when +// no observer survived long enough to journal it (including a first upgrade or +// an abrupt old-server exit). Every open registered session is considered; old +// metadata is not evidence that a long-running task ended. Work is bounded to +// batches of 32 with four concurrent readback probes, never prompt/create/resume. func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) { - prober, ok := broker.(chatTurnProber) - if !ok || h.registry == nil { + if h.registry == nil { return } sessions, err := h.registry.OperatorSessions(ctx) if err != nil { - log.Printf("chat turn recovery: %v", err) + log.Printf("chat recovery registry: %v", err) return } - cutoff := h.clock().Add(-h.unattended.window).UnixMilli() - probed := 0 + ids := make([]string, 0, len(sessions)) for _, session := range sessions { + if session.ClosedAt == 0 && validChatSessionID(session.ID) { + ids = append(ids, session.ID) + } + } + for start := 0; start < len(ids); start += chatRecoverSessionLimit { if ctx.Err() != nil { return } - if session.ClosedAt != 0 || session.UpdatedAt < cutoff || !validChatSessionID(session.ID) { - continue + end := start + chatRecoverSessionLimit + if end > len(ids) { + end = len(ids) } - if probed >= chatRecoverSessionLimit { - break + jobs := make(chan string) + var workers sync.WaitGroup + for i := 0; i < 4; i++ { + workers.Add(1) + go func() { + defer workers.Done() + for id := range jobs { + h.recoverDiscoveredSession(ctx, broker, id) + } + }() } - probed++ - probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - running, err := prober.ProbeTurn(probeCtx, session.ID) - cancel() - if err != nil { - log.Printf("chat turn recovery probe session=%s failed (%T)", session.ID, err) - continue - } - if running { - h.observeUnattended(session.ID, broker) + for _, id := range ids[start:end] { + select { + case jobs <- id: + case <-ctx.Done(): + close(jobs) + workers.Wait() + return + } } + close(jobs) + workers.Wait() } } @@ -245,7 +259,7 @@ func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) { // observes its stream until the turn the harness reported has ended. func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) { h.mu.Lock() - if h.rooms[sessionID] == nil && h.unattendedCountLocked(nil) >= chatUnattendedLimit { + if h.rooms[sessionID] == nil && h.backgroundTasksLocked(sessionID) == 0 && h.unattendedCountLocked(nil) >= chatUnattendedLimit { h.mu.Unlock() log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit) return diff --git a/internal/server/chat_unattended_test.go b/internal/server/chat_unattended_test.go index 9d0d0b6d..d6d2e23d 100644 --- a/internal/server/chat_unattended_test.go +++ b/internal/server/chat_unattended_test.go @@ -279,9 +279,9 @@ func TestChatRunningTurnObservedAfterRestart(t *testing.T) { waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 }) } -// A recovered session whose stream never reports the turn is released after -// the settle window rather than observed forever. -func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) { +// Authoritative busy survives the settle window even without a replayed start; +// a later authoritative idle result releases it. +func TestChatRecoveredBusySessionWaitsForAuthoritativeIdle(t *testing.T) { fastUnattended(t) t.Setenv("PIPELINES_NTFY_URL", "") store := unattendedStore(t, true) @@ -289,7 +289,14 @@ func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) { broker.probe["ses_1"] = true startQueueServer(t, broker, store) waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 }) - waitFor(t, "released after settle", func() bool { active, _ := broker.streams(); return active == 0 }) + time.Sleep(2 * chatUnattendedRecoverSettle) + if active, _ := broker.streams(); active != 1 { + t.Fatal("authoritative busy session lost observer") + } + broker.smu.Lock() + broker.probe["ses_1"] = false + broker.smu.Unlock() + waitFor(t, "released after authoritative idle", func() bool { active, _ := broker.streams(); return active == 0 }) } // A turn that never reports its end does not hold an observer forever. diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 5bcb6d52..633429e8 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -134,9 +134,6 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin // session; resume delivering any a previous process left queued. hub.queueStore = stores[0] hub.liveStore = stores[0] - if options.Chat != nil { - go hub.resumeQueued(ctx, options.Chat) - } } acks := newAttentionAckHub() var ackStore attentionAckStore @@ -500,9 +497,15 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin // that ended while it was down), then keep observing any other chat // whose turn is still running, so a reply or question still notifies // with no client attached. + done := make(chan struct{}) + hub.recoveryDone = done go func() { hub.reattachSessions(ctx, options.Chat) + // Restore live turn ownership before queue rooms can start runners. + hub.resumeQueued(ctx, options.Chat) hub.observeRunningTurns(ctx, options.Chat) + close(done) + hub.retryReattachments(ctx, options.Chat) }() } if runs != nil {