Recover background chat observers from durable live membership

This commit is contained in:
2026-11-07 16:33:49 +00:00
parent 9c67415c22
commit 132a94192a
13 changed files with 700 additions and 59 deletions
+53
View File
@@ -88,3 +88,56 @@ func (s *Store) TakeChatLiveSessions(ctx context.Context) ([]ChatLiveSession, er
}
return sessions, tx.Commit()
}
// ChatLiveSessions reads recovery membership without consuming it. A failed
// reconnect or a second crash must not erase the only record of live work.
func (s *Store) ChatLiveSessions(ctx context.Context) ([]ChatLiveSession, error) {
rows, err := s.db.QueryContext(ctx, `SELECT session_id,backend,turn,native_turn_id,attention,recorded_at FROM chat_live_sessions ORDER BY recorded_at, session_id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []ChatLiveSession
for rows.Next() {
var v ChatLiveSession
var at int64
if err := rows.Scan(&v.SessionID, &v.Backend, &v.Turn, &v.NativeTurnID, &v.Attention, &at); err != nil {
return nil, err
}
v.RecordedAt = time.UnixMilli(at)
out = append(out, v)
}
return out, rows.Err()
}
// RememberChatLiveSession journals live work as it is observed, not only at a
// graceful shutdown. Attention remains sticky until an authoritative reconnect
// confirms idle: evicting a bounded replay cache is not evidence tasks ended.
func (s *Store) RememberChatLiveSession(ctx context.Context, v ChatLiveSession) error {
if v.RecordedAt.IsZero() {
v.RecordedAt = time.Now()
}
if !v.Turn && !v.Attention {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, `UPDATE chat_live_sessions SET turn=0,native_turn_id='' WHERE session_id=?`, v.SessionID); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions WHERE session_id=? AND turn=0 AND attention=0`, v.SessionID); err != nil {
return err
}
return tx.Commit()
}
_, err := s.db.ExecContext(ctx, `INSERT INTO chat_live_sessions(session_id,backend,turn,native_turn_id,attention,recorded_at) VALUES(?,?,?,?,?,?) ON CONFLICT(session_id) DO UPDATE SET backend=excluded.backend,turn=excluded.turn,native_turn_id=CASE WHEN excluded.turn=0 THEN '' WHEN excluded.native_turn_id!='' THEN excluded.native_turn_id ELSE chat_live_sessions.native_turn_id END,attention=MAX(chat_live_sessions.attention,excluded.attention),recorded_at=excluded.recorded_at`, v.SessionID, v.Backend, v.Turn, v.NativeTurnID, v.Attention, v.RecordedAt.UnixMilli())
return err
}
// ForgetChatLiveSession is used only after authoritative idle or explicit close.
func (s *Store) ForgetChatLiveSession(ctx context.Context, id string) error {
_, err := s.db.ExecContext(ctx, `DELETE FROM chat_live_sessions WHERE session_id=?`, id)
return err
}
@@ -0,0 +1,36 @@
package pipeline
import (
"context"
"path/filepath"
"testing"
)
func TestChatRecoveryJournalIsNotConsumedAndKeepsUnverifiedAttention(t *testing.T) {
s, err := Open(filepath.Join(t.TempDir(), "live.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
ctx := context.Background()
if err := s.RememberChatLiveSession(ctx, ChatLiveSession{SessionID: "ses_task", Attention: true}); err != nil {
t.Fatal(err)
}
// Ending the foreground turn/evicting its replay is not task completion.
if err := s.RememberChatLiveSession(ctx, ChatLiveSession{SessionID: "ses_task"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
got, err := s.ChatLiveSessions(ctx)
if err != nil || len(got) != 1 || !got[0].Attention || got[0].Turn {
t.Fatalf("read%d: %+v %v", i, got, err)
}
}
if err := s.ForgetChatLiveSession(ctx, "ses_task"); err != nil {
t.Fatal(err)
}
got, err := s.ChatLiveSessions(ctx)
if err != nil || len(got) != 0 {
t.Fatalf("confirmed idle retained: %+v %v", got, err)
}
}
+1
View File
@@ -53,6 +53,7 @@ func (h *chatHub) noteActivityLocked(sessionID string) {
if sessionID == "" {
return
}
h.rememberLiveLocked(sessionID)
next := h.chatActivityLocked(sessionID)
previous := h.activity[sessionID]
if previous.TurnRunning == next.TurnRunning && previous.BackgroundTasks == next.BackgroundTasks {
+10 -7
View File
@@ -109,10 +109,9 @@ func TestChatActivityReachesEveryConnection(t *testing.T) {
}
}
// A background task that never reports its exit does not hold the observer
// forever: it is released after chatUnattendedTurnLimit, and the list then
// reports no tasks rather than a stale count.
func TestChatUnreportedBackgroundTaskIsReleased(t *testing.T) {
// A running task is not an idle observer: the old observer TTL must not
// invent completion. A terminal event releases it and clears list activity.
func TestChatBackgroundTaskWaitsForExitBeyondObserverTTL(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
limit := chatUnattendedTurnLimit
@@ -134,17 +133,21 @@ func TestChatUnreportedBackgroundTaskIsReleased(t *testing.T) {
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
readActivity(t, server.ctx, list)
viewer.CloseNow()
time.Sleep(100 * time.Millisecond)
time.Sleep(2 * chatUnattendedTurnLimit)
if active, _ := broker.streams(); active != 1 {
t.Fatalf("background task not observed: %d", active)
t.Fatalf("background observer expired: %d", active)
}
if got := listActivity(t, server.ctx, list); got == nil || got.BackgroundTasks != 1 {
t.Fatalf("invented task completion: %+v", got)
}
broker.push(shellFrame("sh_1", protocol.ChatShellExited))
released := readActivity(t, server.ctx, list)
if released.TurnRunning || released.BackgroundTasks != 0 {
t.Fatalf("release push = %+v", released)
}
waitFor(t, "observer released", func() bool { active, _ := broker.streams(); return active == 0 })
if got := listActivity(t, server.ctx, list); got == nil || got.BackgroundTasks != 0 {
t.Fatalf("listed activity after expiry = %+v", got)
t.Fatalf("listed activity after completion = %+v", got)
}
}
@@ -0,0 +1,246 @@
package server
import (
"context"
"database/sql"
"errors"
"fmt"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type recoveryTasks struct {
*restartChat
mu sync.Mutex
tasks map[string][]protocol.ChatShell
unavailable bool
historyReads int
creates int
}
func newRecoveryTasks() *recoveryTasks {
return &recoveryTasks{restartChat: newRestartChat(), tasks: map[string][]protocol.ChatShell{}}
}
func (b *recoveryTasks) Shells(_ context.Context, id string) ([]protocol.ChatShell, error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.unavailable {
return nil, errors.New("temporary read failure")
}
return append([]protocol.ChatShell(nil), b.tasks[id]...), nil
}
func (b *recoveryTasks) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.historyReads++
return protocol.ChatHistoryPage{}, nil
}
func (b *recoveryTasks) Create(context.Context, string, string) (protocol.ChatSession, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.creates++
return protocol.ChatSession{}, errors.New("recovery must not create a session")
}
func task(id string) protocol.ChatShell {
return protocol.ChatShell{ID: id, Status: protocol.ChatShellRunning, Command: "isolated background task", StartedAt: 1234}
}
func waitRecoveredTask(t *testing.T, s *chatRestartServer, id, taskID string) {
t.Helper()
waitFor(t, "background reattached with same identity", func() bool {
h := s.handler.hub
h.mu.Lock()
room := h.rooms[id]
attached := room != nil && room.streaming
h.mu.Unlock()
if !attached {
return false
}
got := h.shellSnapshot(id)
return len(got) == 1 && got[0].ID == taskID && got[0].StartedAt == 1234
})
}
func TestBackgroundRecoverySurvivesRoomEvictionAndUngracefulStop(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
first := newRecoveryTasks()
first.tasks["ses_1"] = []protocol.ChatShell{task("job_same")}
s := startChatRestartServer(t, first, path)
waitRecoveredTask(t, s, "ses_1", "job_same")
h := s.handler.hub
h.mu.Lock()
h.releaseRoomLocked(h.rooms["ses_1"])
h.mu.Unlock()
// No DetachSessions: model a crash after the observer was released.
s.stop()
second := newRecoveryTasks()
second.tasks["ses_1"] = []protocol.ChatShell{task("job_same")}
next := startChatRestartServer(t, second, path)
waitRecoveredTask(t, next, "ses_1", "job_same")
second.mu.Lock()
defer second.mu.Unlock()
if second.historyReads != 0 || second.creates != 0 || second.promptCount() != 0 {
t.Fatal("recovery started/read a conversation instead of reattaching tasks")
}
}
func TestBackgroundRecoveryRetainsJournalAcrossFailedReattachment(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
store := openQueueStore(t, path)
if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}); err != nil {
t.Fatal(err)
}
broken := newRecoveryTasks()
broken.unavailable = true
first := startChatRestartServer(t, broken, path)
select {
case <-first.handler.hub.recoveryDone:
case <-first.ctx.Done():
t.Fatal("recovery did not settle")
}
records, err := store.ChatLiveSessions(context.Background())
if err != nil || len(records) != 1 {
t.Fatalf("failed reconnect consumed journal: %+v %v", records, err)
}
first.stop()
repaired := newRecoveryTasks()
repaired.tasks["ses_1"] = []protocol.ChatShell{task("job_kept")}
next := startChatRestartServer(t, repaired, path)
waitRecoveredTask(t, next, "ses_1", "job_kept")
}
func TestBackgroundRecoveryFindsOldSessionsBeyondFormerRecentLimit(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
store := openQueueStore(t, path)
for i := 0; i < chatRecoverSessionLimit+5; i++ {
if err := store.RecordOperatorSession(context.Background(), fmt.Sprintf("ses_new_%02d", i), t.TempDir(), "idle", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
}
db, err := sql.Open("sqlite", path)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Exec(`UPDATE operator_sessions SET last_used_at=? WHERE session_id='ses_1'`, time.Now().Add(-7*24*time.Hour).UnixMilli()); err != nil {
t.Fatal(err)
}
broker := newRecoveryTasks()
broker.tasks["ses_1"] = []protocol.ChatShell{task("job_old")}
s := startChatRestartServer(t, broker, path)
waitRecoveredTask(t, s, "ses_1", "job_old")
broker.mu.Lock()
defer broker.mu.Unlock()
if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 {
t.Fatal("discovery resumed or read chat history")
}
}
func TestBackgroundRecoveryKeepsLiveObserversBeyondIdleCapAndTTL(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
store := openQueueStore(t, path)
broker := newRecoveryTasks()
count := chatUnattendedLimit + 3
for i := 0; i < count; i++ {
id := fmt.Sprintf("ses_busy_%02d", i)
if err := store.RecordOperatorSession(context.Background(), id, t.TempDir(), "background", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
broker.tasks[id] = []protocol.ChatShell{task("job_" + id)}
}
s := startChatRestartServer(t, broker, path)
waitFor(t, "all background streams restored despite idle cap", func() bool { active, _ := broker.streams(); return active == count })
h := s.handler.hub
h.mu.Lock()
rooms := make([]*chatRoom, 0, len(h.rooms))
for _, room := range h.rooms {
room.unattendedSince = h.clock().Add(-7 * time.Hour)
rooms = append(rooms, room)
}
h.mu.Unlock()
for _, room := range rooms {
if h.reapUnattended(room) {
t.Fatal("live task observer expired")
}
}
if active, _ := broker.streams(); active != count {
t.Fatalf("lost background streams: %d", active)
}
}
// A readback captured before a newer task event must not erase that task's
// durable recovery membership, even if its observer has since been released.
type staleRecoveryTasks struct {
*recoveryTasks
entered chan struct{}
release chan struct{}
}
func (b *staleRecoveryTasks) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
close(b.entered)
select {
case <-b.release:
return nil, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func TestBackgroundRecoveryStaleIdleCannotForgetNewTask(t *testing.T) {
ctx := context.Background()
store := openQueueStore(t, restartChatDB(t))
old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}
if err := store.RememberChatLiveSession(ctx, old); err != nil {
t.Fatal(err)
}
h := newChatHub()
h.liveStore = store
broker := &staleRecoveryTasks{recoveryTasks: newRecoveryTasks(), entered: make(chan struct{}), release: make(chan struct{})}
done := make(chan struct{})
go func() { h.reattachSession(ctx, broker, old); close(done) }()
<-broker.entered
h.mu.Lock()
room := h.roomLocked("ses_1")
newest := task("new_job")
h.applyLocked(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &newest})
h.releaseRoomLocked(room)
h.mu.Unlock()
close(broker.release)
<-done
rows, err := store.ChatLiveSessions(ctx)
if err != nil {
t.Fatal(err)
}
if len(rows) != 1 || !rows[0].Attention {
t.Fatalf("stale idle erased new task journal: %+v", rows)
}
}
func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T) {
ctx := context.Background()
store := openQueueStore(t, restartChatDB(t))
old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}
if err := store.RememberChatLiveSession(ctx, old); err != nil {
t.Fatal(err)
}
h := newChatHub()
h.liveStore = store
broker := newRecoveryTasks()
h.reattachSession(ctx, broker, old)
rows, err := store.ChatLiveSessions(ctx)
if err != nil {
t.Fatal(err)
}
if len(rows) != 0 {
t.Fatalf("authoritative idle left stale journal: %+v", rows)
}
if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 {
t.Fatal("idle recovery started or read a conversation")
}
}
+82 -7
View File
@@ -28,11 +28,12 @@ import (
// OpenCode (an attached or detached owned service), Codex (its App Server) and
// Claude Code (its detachable hosts) implement Detach.
// chatLiveStore records, on shutdown, the chats left live on their harness so
// the next start reattaches exactly those; *pipeline.Store implements it.
// chatLiveStore durably journals live work before room eviction/crash, as well
// as on graceful shutdown; *pipeline.Store implements it.
type chatLiveStore interface {
RecordChatLiveSessions(ctx context.Context, sessions []pipeline.ChatLiveSession) error
TakeChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
ChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
RememberChatLiveSession(ctx context.Context, session pipeline.ChatLiveSession) error
ForgetChatLiveSession(ctx context.Context, id string) error
}
// chatReattachPoll is how often a reattached turn re-asks its harness whether
@@ -63,10 +64,15 @@ func (h *chatHub) detachSessions(ctx context.Context) error {
if len(live) > 0 {
log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live))
}
return h.liveStore.RecordChatLiveSessions(ctx, live)
for _, session := range live {
if err := h.liveStore.RememberChatLiveSession(ctx, session); err != nil {
return err
}
}
return nil
}
// reattachSessions is the start-up half of detachSessions: each recorded chat
// reattachSessions reads (without consuming) the recovery journal. Each chat
// is observed again (stream, pending questions, background tasks) while its
// harness still has work, and a turn that ended while the server was down is
// folded in from the harness's history, so its reply still notifies and its
@@ -75,7 +81,7 @@ func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
if h.liveStore == nil || broker == nil {
return
}
live, err := h.liveStore.TakeChatLiveSessions(ctx)
live, err := h.liveStore.ChatLiveSessions(ctx)
if err != nil {
log.Printf("chat reattach: %v", err)
return
@@ -87,13 +93,34 @@ func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
if !validChatSessionID(session.SessionID) {
continue
}
h.mu.Lock()
observed := h.rooms[session.SessionID] != nil
generation := h.recoveryGeneration[session.SessionID]
h.mu.Unlock()
if observed {
continue
}
if h.registry != nil {
row, found, err := h.registry.OperatorSession(ctx, session.SessionID)
if err != nil {
continue
}
if !found || row.ClosedAt != 0 {
h.forgetLive(ctx, session.SessionID, generation)
continue
}
}
h.reattachSession(ctx, broker, session)
}
}
func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) {
id := session.SessionID
h.mu.Lock()
generation := h.recoveryGeneration[id]
h.mu.Unlock()
running := false
turnKnown := false
if prober, ok := broker.(chatTurnProber); ok {
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
busy, err := prober.ProbeTurn(probeCtx, id)
@@ -104,6 +131,54 @@ func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, sessio
busy = session.Turn
}
running = busy
turnKnown = err == nil
}
var snapshot []protocol.ChatShell
attentionKnown := false
if !running {
// A recovered task may have ended while disconnected. Only clear the
// journal after all pending-work surfaces authoritatively report idle.
if attention, known, shells := probeChatAttention(ctx, broker, id); known {
snapshot = shells
attentionKnown = true
session.Attention = attention
} else {
session.Attention = true // Unknown pending work is not idle.
}
}
h.mu.Lock()
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
if !stale && attentionKnown {
h.replaceShellsLocked(id, snapshot)
}
h.mu.Unlock()
if stale {
return
}
if turnKnown && !running && !session.Attention {
if session.Turn {
h.foldEndedTurn(ctx, broker, session, nil)
}
if h.liveStore != nil {
h.forgetLive(ctx, id, generation)
}
return
}
retained := session
retained.Turn = running
retained.RecordedAt = h.clock()
if !h.rememberRecovered(ctx, retained, generation) {
return
}
if running && session.Turn {
// Seed ownership before opening the stream: replay/poll admission can
// journal immediately and must not overwrite the recovered running turn.
h.mu.Lock()
turn := h.turnLocked(id)
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
}
h.mu.Unlock()
}
if running || session.Attention {
h.observeUnattended(id, broker)
+9 -1
View File
@@ -56,7 +56,10 @@ type chatHub struct {
queueStore chatPromptQueueStore
// liveStore records the chats a stopping server leaves live on their
// harness (chat_detach.go); nil reattaches nothing beyond the start probe.
liveStore chatLiveStore
liveStore chatLiveStore
liveJournal map[string]chatRecoveryState
recoveryGeneration map[string]uint64
recoveryDone <-chan struct{}
// recentOps/recentOpOrder are the bounded record of accepted prompt
// operation ids, so a client retry is acknowledged instead of re-sent.
recentOps map[string]chatRecentOp
@@ -202,7 +205,11 @@ func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
if room := h.rooms[frame.SessionID]; room != nil {
room.shellReadback.Record(frame.Shell)
}
if frame.Shell != nil && frame.Shell.ID != "" {
h.touchRecoveryLocked(frame.SessionID)
}
turn.applyShell(frame.Shell)
h.rememberLiveLocked(frame.SessionID)
return
case protocol.ChatEventForm:
// A form/question likewise carries no message id. Keep the pending form
@@ -221,6 +228,7 @@ func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
if !frame.Form.Resolved && !frame.Form.Cancelled && len(room.forms) < 8 {
room.forms = append(room.forms, *frame.Form)
}
h.rememberLiveLocked(frame.SessionID)
}
return
}
+187
View File
@@ -0,0 +1,187 @@
package server
import (
"context"
"log"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type chatRecoveryState struct {
turn, attention bool
native string
}
// rememberLiveLocked journals liveness before an observer can be evicted or
// the process can crash. A bounded replay cache is never the recovery journal.
// Callers hold h.mu; like durable prompt admission, this small SQLite write is
// ordered with the lifecycle change it describes.
func (h *chatHub) rememberLiveLocked(id string) {
if h.liveStore == nil || (h.baseCtx != nil && h.baseCtx.Err() != nil) {
return
}
room := h.rooms[id]
if room == nil {
return
} // Losing observation is not authoritative idle.
turn := room.running || room.observedTurn
native := room.nativeTurnID
if current := h.turns[id]; current != nil && current.step != nil && current.step.Status == protocol.ChatStepStarted && !current.step.Intermediate {
turn = true
if native == "" {
native = current.step.TurnID
}
}
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
next := chatRecoveryState{turn: turn, attention: attention, native: native}
if previous, ok := h.liveJournal[id]; ok && previous == next {
return
}
h.touchRecoveryLocked(id)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := h.liveStore.RememberChatLiveSession(ctx, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: native, Attention: attention, RecordedAt: h.clock()}); err != nil {
log.Printf("chat recovery journal session=%s failed (%T)", id, err)
return
}
if h.liveJournal == nil {
h.liveJournal = map[string]chatRecoveryState{}
}
h.liveJournal[id] = next
}
func (h *chatHub) touchRecoveryLocked(id string) {
if h.recoveryGeneration == nil {
h.recoveryGeneration = map[string]uint64{}
}
h.recoveryGeneration[id]++
}
// Probe results are older than any lifecycle that arrived while they were in
// flight. Hold the same lock as lifecycle journaling through the final delete.
func (h *chatHub) forgetLive(ctx context.Context, id string, generation uint64) {
if h.liveStore == nil {
return
}
h.mu.Lock()
defer h.mu.Unlock()
if h.recoveryGeneration[id] != generation || h.rooms[id] != nil || ctx.Err() != nil {
return
}
if err := h.liveStore.ForgetChatLiveSession(ctx, id); err != nil {
log.Printf("chat recovery clear session=%s failed (%T)", id, err)
return
}
delete(h.liveJournal, id)
}
func (h *chatHub) rememberRecovered(ctx context.Context, v pipeline.ChatLiveSession, generation uint64) bool {
h.mu.Lock()
defer h.mu.Unlock()
if h.recoveryGeneration[v.SessionID] != generation || h.rooms[v.SessionID] != nil || ctx.Err() != nil {
return false
}
h.touchRecoveryLocked(v.SessionID)
if h.liveStore != nil {
if err := h.liveStore.RememberChatLiveSession(ctx, v); err != nil {
log.Printf("chat reattach journal session=%s failed (%T)", v.SessionID, err)
}
}
return true
}
// probeChatAttention reads only authoritative pending-work surfaces, never
// starts a model turn or infers task completion from absence of foreground work.
func probeChatAttention(ctx context.Context, broker chatBroker, id string) (attention, known bool, shells []protocol.ChatShell) {
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
shells, err := broker.Shells(probe, id)
if err != nil {
return false, false, nil
}
for _, shell := range shells {
if shell.Status == protocol.ChatShellRunning {
return true, true, shells
}
}
permissions, err := broker.Permissions(probe, id)
if err != nil {
return false, false, shells
}
for _, p := range permissions {
if !p.Resolved {
return true, true, shells
}
}
if reader, ok := broker.(chatFormReader); ok {
forms, err := reader.Forms(probe, id)
if err != nil {
return false, false, shells
}
for _, f := range forms {
if !f.Resolved && !f.Cancelled {
return true, true, shells
}
}
}
return false, true, shells
}
const chatRecoveryRetry = 30 * time.Second
// Only explicitly journaled but unobserved sessions are retried. Transient
// harness unavailability must survive more than one start, without polling all
// historical chats repeatedly.
func (h *chatHub) retryReattachments(ctx context.Context, broker chatBroker) {
ticker := time.NewTicker(chatRecoveryRetry)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
h.reattachSessions(ctx, broker)
}
}
}
// recoverDiscoveredSession attaches only if the harness reports actual work.
// Successful foreground-idle alone never rules out background tasks/questions.
func (h *chatHub) recoverDiscoveredSession(ctx context.Context, broker chatBroker, id string) {
h.mu.Lock()
observed := h.rooms[id] != nil
generation := h.recoveryGeneration[id]
h.mu.Unlock()
if observed || ctx.Err() != nil {
return
}
running := false
if prober, ok := broker.(chatTurnProber); ok {
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
busy, err := prober.ProbeTurn(probe, id)
cancel()
if err == nil {
running = busy
}
}
attention, known, shells := probeChatAttention(ctx, broker, id)
h.mu.Lock()
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
if !stale && known {
h.replaceShellsLocked(id, shells)
}
h.mu.Unlock()
if stale {
return
}
if !running && !attention {
return
}
session := pipeline.ChatLiveSession{SessionID: id, Turn: running, Attention: attention, RecordedAt: h.clock()}
if !h.rememberRecovered(ctx, session, generation) {
return
}
h.reattachSession(ctx, broker, session)
}
+2
View File
@@ -343,6 +343,7 @@ func (h *chatHub) releaseRoomLocked(room *chatRoom) {
if h.rooms[room.sessionID] != room {
return
}
h.rememberLiveLocked(room.sessionID)
h.clearCompactionLocked(room)
delete(h.rooms, room.sessionID)
room.cancel()
@@ -709,6 +710,7 @@ func (h *chatHub) pushPermission(sessionID string, frame protocol.ChatPermission
}
}
room.pending = append(room.pending, frame)
h.rememberLiveLocked(sessionID)
h.notePermissionAttentionLocked(frame)
if len(room.pending) > chatRoomPendingLimit {
room.pending = room.pending[len(room.pending)-chatRoomPendingLimit:]
+7 -1
View File
@@ -85,7 +85,13 @@ func testChatShellOutputOwnedAndForeign(t *testing.T, exited bool) {
if err := store.RecordOperatorSession(ctx, LargeChatFixtureSessionID, t.TempDir(), "owned", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store}))
handler := HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store}).(*runHandler)
srv := httptest.NewServer(handler)
select {
case <-handler.hub.recoveryDone:
case <-ctx.Done():
t.Fatal("startup recovery did not finish")
}
defer srv.Close()
ws := dialShellOutputWebsocket(t, ctx, srv)
defer ws.CloseNow()
+50 -36
View File
@@ -3,6 +3,7 @@ package server
import (
"context"
"log"
"sync"
"time"
)
@@ -17,11 +18,10 @@ import (
// observation is started at server start for sessions whose harness reports a
// running turn, so a reply that finishes after a restart still notifies.
//
// Resources are bounded: at most chatUnattendedLimit rooms are kept without a
// client (rooms holding queued prompts are not counted: they were always kept
// until delivered), a stuck turn or a background task that never reports an
// exit is released after chatUnattendedTurnLimit, and an idle session is never
// observed.
// Idle/stale foreground observers are bounded by chatUnattendedLimit and
// chatUnattendedTurnLimit. Known running background work (like queued prompts)
// is not an idle lease: it remains attached until actual completion or explicit
// stop. Its durable journal is independent of bounded replay/room retention.
// chatUnattendedLimit bounds the rooms kept observing with no client attached
// and no queued prompt.
@@ -73,10 +73,8 @@ func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool {
if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 {
return true
}
// Running background tasks keep the stream observed so their exits are
// seen and the chat list's indicator clears. This is bounded like a turn:
// the unattended room limit, the chatUnattendedTurnLimit expiry, and the
// harness itself (a Claude Code process that exits ends its tasks).
// Running background tasks keep the stream observed so their actual exits
// are seen and the chat list's indicator clears, without a fixed age cap.
if h.backgroundTasksLocked(room.sessionID) > 0 {
return true
}
@@ -108,7 +106,7 @@ func (h *chatHub) leaveUnattendedLocked(room *chatRoom) {
h.releaseRoomLocked(room)
return
}
if !room.watching && h.unattendedCountLocked(room) >= chatUnattendedLimit {
if !room.watching && h.backgroundTasksLocked(room.sessionID) == 0 && h.unattendedCountLocked(room) >= chatUnattendedLimit {
log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit)
h.releaseRoomLocked(room)
return
@@ -172,6 +170,11 @@ func (h *chatHub) reapUnattended(room *chatRoom) bool {
room.unattendedSince = h.clock()
return false
}
// A running task is actual work, not an idle-observer lease. Its terminal
// lifecycle/readback, or explicit operator stop, releases it normally.
if h.backgroundTasksLocked(room.sessionID) > 0 {
return false
}
expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit
if !expired && h.keepUnattendedLocked(room, busy, true) {
return false
@@ -199,45 +202,56 @@ func (h *chatHub) attendedLocked(room *chatRoom) bool {
return false
}
// observeRunningTurns is the start-up half: each recently used open operator
// session whose harness reports a turn in flight gets an unattended room
// observing its stream, so the reply (or a question) still notifies. It is
// best effort per harness: OpenCode and Codex keep running across a Pipelines
// restart and report it; Claude Code runs as a child of the server and so has
// no turn left after a restart.
// observeRunningTurns discovers live foreground or background work even when
// no observer survived long enough to journal it (including a first upgrade or
// an abrupt old-server exit). Every open registered session is considered; old
// metadata is not evidence that a long-running task ended. Work is bounded to
// batches of 32 with four concurrent readback probes, never prompt/create/resume.
func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
prober, ok := broker.(chatTurnProber)
if !ok || h.registry == nil {
if h.registry == nil {
return
}
sessions, err := h.registry.OperatorSessions(ctx)
if err != nil {
log.Printf("chat turn recovery: %v", err)
log.Printf("chat recovery registry: %v", err)
return
}
cutoff := h.clock().Add(-h.unattended.window).UnixMilli()
probed := 0
ids := make([]string, 0, len(sessions))
for _, session := range sessions {
if session.ClosedAt == 0 && validChatSessionID(session.ID) {
ids = append(ids, session.ID)
}
}
for start := 0; start < len(ids); start += chatRecoverSessionLimit {
if ctx.Err() != nil {
return
}
if session.ClosedAt != 0 || session.UpdatedAt < cutoff || !validChatSessionID(session.ID) {
continue
end := start + chatRecoverSessionLimit
if end > len(ids) {
end = len(ids)
}
if probed >= chatRecoverSessionLimit {
break
jobs := make(chan string)
var workers sync.WaitGroup
for i := 0; i < 4; i++ {
workers.Add(1)
go func() {
defer workers.Done()
for id := range jobs {
h.recoverDiscoveredSession(ctx, broker, id)
}
}()
}
probed++
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
running, err := prober.ProbeTurn(probeCtx, session.ID)
cancel()
if err != nil {
log.Printf("chat turn recovery probe session=%s failed (%T)", session.ID, err)
continue
}
if running {
h.observeUnattended(session.ID, broker)
for _, id := range ids[start:end] {
select {
case jobs <- id:
case <-ctx.Done():
close(jobs)
workers.Wait()
return
}
}
close(jobs)
workers.Wait()
}
}
@@ -245,7 +259,7 @@ func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
// observes its stream until the turn the harness reported has ended.
func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
h.mu.Lock()
if h.rooms[sessionID] == nil && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
if h.rooms[sessionID] == nil && h.backgroundTasksLocked(sessionID) == 0 && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
h.mu.Unlock()
log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit)
return
+11 -4
View File
@@ -279,9 +279,9 @@ func TestChatRunningTurnObservedAfterRestart(t *testing.T) {
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A recovered session whose stream never reports the turn is released after
// the settle window rather than observed forever.
func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) {
// Authoritative busy survives the settle window even without a replayed start;
// a later authoritative idle result releases it.
func TestChatRecoveredBusySessionWaitsForAuthoritativeIdle(t *testing.T) {
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
store := unattendedStore(t, true)
@@ -289,7 +289,14 @@ func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) {
broker.probe["ses_1"] = true
startQueueServer(t, broker, store)
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
waitFor(t, "released after settle", func() bool { active, _ := broker.streams(); return active == 0 })
time.Sleep(2 * chatUnattendedRecoverSettle)
if active, _ := broker.streams(); active != 1 {
t.Fatal("authoritative busy session lost observer")
}
broker.smu.Lock()
broker.probe["ses_1"] = false
broker.smu.Unlock()
waitFor(t, "released after authoritative idle", func() bool { active, _ := broker.streams(); return active == 0 })
}
// A turn that never reports its end does not hold an observer forever.
+6 -3
View File
@@ -134,9 +134,6 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
// session; resume delivering any a previous process left queued.
hub.queueStore = stores[0]
hub.liveStore = stores[0]
if options.Chat != nil {
go hub.resumeQueued(ctx, options.Chat)
}
}
acks := newAttentionAckHub()
var ackStore attentionAckStore
@@ -500,9 +497,15 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
// that ended while it was down), then keep observing any other chat
// whose turn is still running, so a reply or question still notifies
// with no client attached.
done := make(chan struct{})
hub.recoveryDone = done
go func() {
hub.reattachSessions(ctx, options.Chat)
// Restore live turn ownership before queue rooms can start runners.
hub.resumeQueued(ctx, options.Chat)
hub.observeRunningTurns(ctx, options.Chat)
close(done)
hub.retryReattachments(ctx, options.Chat)
}()
}
if runs != nil {