Recover background chat observers from durable live membership
This commit is contained in:
@@ -88,3 +88,56 @@ func (s *Store) TakeChatLiveSessions(ctx context.Context) ([]ChatLiveSession, er
|
||||
}
|
||||
return sessions, tx.Commit()
|
||||
}
|
||||
|
||||
// ChatLiveSessions reads recovery membership without consuming it. A failed
|
||||
// reconnect or a second crash must not erase the only record of live work.
|
||||
func (s *Store) ChatLiveSessions(ctx context.Context) ([]ChatLiveSession, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT session_id,backend,turn,native_turn_id,attention,recorded_at FROM chat_live_sessions ORDER BY recorded_at, session_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ChatLiveSession
|
||||
for rows.Next() {
|
||||
var v ChatLiveSession
|
||||
var at int64
|
||||
if err := rows.Scan(&v.SessionID, &v.Backend, &v.Turn, &v.NativeTurnID, &v.Attention, &at); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v.RecordedAt = time.UnixMilli(at)
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RememberChatLiveSession journals live work as it is observed, not only at a
|
||||
// graceful shutdown. Attention remains sticky until an authoritative reconnect
|
||||
// confirms idle: evicting a bounded replay cache is not evidence tasks ended.
|
||||
func (s *Store) RememberChatLiveSession(ctx context.Context, v ChatLiveSession) error {
|
||||
if v.RecordedAt.IsZero() {
|
||||
v.RecordedAt = time.Now()
|
||||
}
|
||||
if !v.Turn && !v.Attention {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE chat_live_sessions SET turn=0,native_turn_id='' WHERE session_id=?`, v.SessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions WHERE session_id=? AND turn=0 AND attention=0`, v.SessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
_, err := s.db.ExecContext(ctx, `INSERT INTO chat_live_sessions(session_id,backend,turn,native_turn_id,attention,recorded_at) VALUES(?,?,?,?,?,?) ON CONFLICT(session_id) DO UPDATE SET backend=excluded.backend,turn=excluded.turn,native_turn_id=CASE WHEN excluded.turn=0 THEN '' WHEN excluded.native_turn_id!='' THEN excluded.native_turn_id ELSE chat_live_sessions.native_turn_id END,attention=MAX(chat_live_sessions.attention,excluded.attention),recorded_at=excluded.recorded_at`, v.SessionID, v.Backend, v.Turn, v.NativeTurnID, v.Attention, v.RecordedAt.UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
// ForgetChatLiveSession is used only after authoritative idle or explicit close.
|
||||
func (s *Store) ForgetChatLiveSession(ctx context.Context, id string) error {
|
||||
_, err := s.db.ExecContext(ctx, `DELETE FROM chat_live_sessions WHERE session_id=?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestChatRecoveryJournalIsNotConsumedAndKeepsUnverifiedAttention(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "live.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
ctx := context.Background()
|
||||
if err := s.RememberChatLiveSession(ctx, ChatLiveSession{SessionID: "ses_task", Attention: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Ending the foreground turn/evicting its replay is not task completion.
|
||||
if err := s.RememberChatLiveSession(ctx, ChatLiveSession{SessionID: "ses_task"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
got, err := s.ChatLiveSessions(ctx)
|
||||
if err != nil || len(got) != 1 || !got[0].Attention || got[0].Turn {
|
||||
t.Fatalf("read%d: %+v %v", i, got, err)
|
||||
}
|
||||
}
|
||||
if err := s.ForgetChatLiveSession(ctx, "ses_task"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.ChatLiveSessions(ctx)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("confirmed idle retained: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,7 @@ func (h *chatHub) noteActivityLocked(sessionID string) {
|
||||
if sessionID == "" {
|
||||
return
|
||||
}
|
||||
h.rememberLiveLocked(sessionID)
|
||||
next := h.chatActivityLocked(sessionID)
|
||||
previous := h.activity[sessionID]
|
||||
if previous.TurnRunning == next.TurnRunning && previous.BackgroundTasks == next.BackgroundTasks {
|
||||
|
||||
@@ -109,10 +109,9 @@ func TestChatActivityReachesEveryConnection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A background task that never reports its exit does not hold the observer
|
||||
// forever: it is released after chatUnattendedTurnLimit, and the list then
|
||||
// reports no tasks rather than a stale count.
|
||||
func TestChatUnreportedBackgroundTaskIsReleased(t *testing.T) {
|
||||
// A running task is not an idle observer: the old observer TTL must not
|
||||
// invent completion. A terminal event releases it and clears list activity.
|
||||
func TestChatBackgroundTaskWaitsForExitBeyondObserverTTL(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
limit := chatUnattendedTurnLimit
|
||||
@@ -134,17 +133,21 @@ func TestChatUnreportedBackgroundTaskIsReleased(t *testing.T) {
|
||||
broker.push(stepFrameFor(protocol.ChatStepEnded, "turn_1"))
|
||||
readActivity(t, server.ctx, list)
|
||||
viewer.CloseNow()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
time.Sleep(2 * chatUnattendedTurnLimit)
|
||||
if active, _ := broker.streams(); active != 1 {
|
||||
t.Fatalf("background task not observed: %d", active)
|
||||
t.Fatalf("background observer expired: %d", active)
|
||||
}
|
||||
if got := listActivity(t, server.ctx, list); got == nil || got.BackgroundTasks != 1 {
|
||||
t.Fatalf("invented task completion: %+v", got)
|
||||
}
|
||||
broker.push(shellFrame("sh_1", protocol.ChatShellExited))
|
||||
released := readActivity(t, server.ctx, list)
|
||||
if released.TurnRunning || released.BackgroundTasks != 0 {
|
||||
t.Fatalf("release push = %+v", released)
|
||||
}
|
||||
waitFor(t, "observer released", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
if got := listActivity(t, server.ctx, list); got == nil || got.BackgroundTasks != 0 {
|
||||
t.Fatalf("listed activity after expiry = %+v", got)
|
||||
t.Fatalf("listed activity after completion = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
type recoveryTasks struct {
|
||||
*restartChat
|
||||
mu sync.Mutex
|
||||
tasks map[string][]protocol.ChatShell
|
||||
unavailable bool
|
||||
historyReads int
|
||||
creates int
|
||||
}
|
||||
|
||||
func newRecoveryTasks() *recoveryTasks {
|
||||
return &recoveryTasks{restartChat: newRestartChat(), tasks: map[string][]protocol.ChatShell{}}
|
||||
}
|
||||
func (b *recoveryTasks) Shells(_ context.Context, id string) ([]protocol.ChatShell, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.unavailable {
|
||||
return nil, errors.New("temporary read failure")
|
||||
}
|
||||
return append([]protocol.ChatShell(nil), b.tasks[id]...), nil
|
||||
}
|
||||
func (b *recoveryTasks) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.historyReads++
|
||||
return protocol.ChatHistoryPage{}, nil
|
||||
}
|
||||
func (b *recoveryTasks) Create(context.Context, string, string) (protocol.ChatSession, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.creates++
|
||||
return protocol.ChatSession{}, errors.New("recovery must not create a session")
|
||||
}
|
||||
func task(id string) protocol.ChatShell {
|
||||
return protocol.ChatShell{ID: id, Status: protocol.ChatShellRunning, Command: "isolated background task", StartedAt: 1234}
|
||||
}
|
||||
func waitRecoveredTask(t *testing.T, s *chatRestartServer, id, taskID string) {
|
||||
t.Helper()
|
||||
waitFor(t, "background reattached with same identity", func() bool {
|
||||
h := s.handler.hub
|
||||
h.mu.Lock()
|
||||
room := h.rooms[id]
|
||||
attached := room != nil && room.streaming
|
||||
h.mu.Unlock()
|
||||
if !attached {
|
||||
return false
|
||||
}
|
||||
got := h.shellSnapshot(id)
|
||||
return len(got) == 1 && got[0].ID == taskID && got[0].StartedAt == 1234
|
||||
})
|
||||
}
|
||||
|
||||
func TestBackgroundRecoverySurvivesRoomEvictionAndUngracefulStop(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
first := newRecoveryTasks()
|
||||
first.tasks["ses_1"] = []protocol.ChatShell{task("job_same")}
|
||||
s := startChatRestartServer(t, first, path)
|
||||
waitRecoveredTask(t, s, "ses_1", "job_same")
|
||||
h := s.handler.hub
|
||||
h.mu.Lock()
|
||||
h.releaseRoomLocked(h.rooms["ses_1"])
|
||||
h.mu.Unlock()
|
||||
// No DetachSessions: model a crash after the observer was released.
|
||||
s.stop()
|
||||
second := newRecoveryTasks()
|
||||
second.tasks["ses_1"] = []protocol.ChatShell{task("job_same")}
|
||||
next := startChatRestartServer(t, second, path)
|
||||
waitRecoveredTask(t, next, "ses_1", "job_same")
|
||||
second.mu.Lock()
|
||||
defer second.mu.Unlock()
|
||||
if second.historyReads != 0 || second.creates != 0 || second.promptCount() != 0 {
|
||||
t.Fatal("recovery started/read a conversation instead of reattaching tasks")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackgroundRecoveryRetainsJournalAcrossFailedReattachment(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
store := openQueueStore(t, path)
|
||||
if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
broken := newRecoveryTasks()
|
||||
broken.unavailable = true
|
||||
first := startChatRestartServer(t, broken, path)
|
||||
select {
|
||||
case <-first.handler.hub.recoveryDone:
|
||||
case <-first.ctx.Done():
|
||||
t.Fatal("recovery did not settle")
|
||||
}
|
||||
records, err := store.ChatLiveSessions(context.Background())
|
||||
if err != nil || len(records) != 1 {
|
||||
t.Fatalf("failed reconnect consumed journal: %+v %v", records, err)
|
||||
}
|
||||
first.stop()
|
||||
repaired := newRecoveryTasks()
|
||||
repaired.tasks["ses_1"] = []protocol.ChatShell{task("job_kept")}
|
||||
next := startChatRestartServer(t, repaired, path)
|
||||
waitRecoveredTask(t, next, "ses_1", "job_kept")
|
||||
}
|
||||
|
||||
func TestBackgroundRecoveryFindsOldSessionsBeyondFormerRecentLimit(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
store := openQueueStore(t, path)
|
||||
for i := 0; i < chatRecoverSessionLimit+5; i++ {
|
||||
if err := store.RecordOperatorSession(context.Background(), fmt.Sprintf("ses_new_%02d", i), t.TempDir(), "idle", protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(`UPDATE operator_sessions SET last_used_at=? WHERE session_id='ses_1'`, time.Now().Add(-7*24*time.Hour).UnixMilli()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
broker := newRecoveryTasks()
|
||||
broker.tasks["ses_1"] = []protocol.ChatShell{task("job_old")}
|
||||
s := startChatRestartServer(t, broker, path)
|
||||
waitRecoveredTask(t, s, "ses_1", "job_old")
|
||||
broker.mu.Lock()
|
||||
defer broker.mu.Unlock()
|
||||
if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 {
|
||||
t.Fatal("discovery resumed or read chat history")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackgroundRecoveryKeepsLiveObserversBeyondIdleCapAndTTL(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
store := openQueueStore(t, path)
|
||||
broker := newRecoveryTasks()
|
||||
count := chatUnattendedLimit + 3
|
||||
for i := 0; i < count; i++ {
|
||||
id := fmt.Sprintf("ses_busy_%02d", i)
|
||||
if err := store.RecordOperatorSession(context.Background(), id, t.TempDir(), "background", protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
broker.tasks[id] = []protocol.ChatShell{task("job_" + id)}
|
||||
}
|
||||
s := startChatRestartServer(t, broker, path)
|
||||
waitFor(t, "all background streams restored despite idle cap", func() bool { active, _ := broker.streams(); return active == count })
|
||||
h := s.handler.hub
|
||||
h.mu.Lock()
|
||||
rooms := make([]*chatRoom, 0, len(h.rooms))
|
||||
for _, room := range h.rooms {
|
||||
room.unattendedSince = h.clock().Add(-7 * time.Hour)
|
||||
rooms = append(rooms, room)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, room := range rooms {
|
||||
if h.reapUnattended(room) {
|
||||
t.Fatal("live task observer expired")
|
||||
}
|
||||
}
|
||||
if active, _ := broker.streams(); active != count {
|
||||
t.Fatalf("lost background streams: %d", active)
|
||||
}
|
||||
}
|
||||
|
||||
// A readback captured before a newer task event must not erase that task's
|
||||
// durable recovery membership, even if its observer has since been released.
|
||||
type staleRecoveryTasks struct {
|
||||
*recoveryTasks
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (b *staleRecoveryTasks) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
|
||||
close(b.entered)
|
||||
select {
|
||||
case <-b.release:
|
||||
return nil, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
func TestBackgroundRecoveryStaleIdleCannotForgetNewTask(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := openQueueStore(t, restartChatDB(t))
|
||||
old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}
|
||||
if err := store.RememberChatLiveSession(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := newChatHub()
|
||||
h.liveStore = store
|
||||
broker := &staleRecoveryTasks{recoveryTasks: newRecoveryTasks(), entered: make(chan struct{}), release: make(chan struct{})}
|
||||
done := make(chan struct{})
|
||||
go func() { h.reattachSession(ctx, broker, old); close(done) }()
|
||||
<-broker.entered
|
||||
h.mu.Lock()
|
||||
room := h.roomLocked("ses_1")
|
||||
newest := task("new_job")
|
||||
h.applyLocked(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &newest})
|
||||
h.releaseRoomLocked(room)
|
||||
h.mu.Unlock()
|
||||
close(broker.release)
|
||||
<-done
|
||||
rows, err := store.ChatLiveSessions(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 || !rows[0].Attention {
|
||||
t.Fatalf("stale idle erased new task journal: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := openQueueStore(t, restartChatDB(t))
|
||||
old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}
|
||||
if err := store.RememberChatLiveSession(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := newChatHub()
|
||||
h.liveStore = store
|
||||
broker := newRecoveryTasks()
|
||||
h.reattachSession(ctx, broker, old)
|
||||
rows, err := store.ChatLiveSessions(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("authoritative idle left stale journal: %+v", rows)
|
||||
}
|
||||
if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 {
|
||||
t.Fatal("idle recovery started or read a conversation")
|
||||
}
|
||||
}
|
||||
@@ -28,11 +28,12 @@ import (
|
||||
// OpenCode (an attached or detached owned service), Codex (its App Server) and
|
||||
// Claude Code (its detachable hosts) implement Detach.
|
||||
|
||||
// chatLiveStore records, on shutdown, the chats left live on their harness so
|
||||
// the next start reattaches exactly those; *pipeline.Store implements it.
|
||||
// chatLiveStore durably journals live work before room eviction/crash, as well
|
||||
// as on graceful shutdown; *pipeline.Store implements it.
|
||||
type chatLiveStore interface {
|
||||
RecordChatLiveSessions(ctx context.Context, sessions []pipeline.ChatLiveSession) error
|
||||
TakeChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
|
||||
ChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
|
||||
RememberChatLiveSession(ctx context.Context, session pipeline.ChatLiveSession) error
|
||||
ForgetChatLiveSession(ctx context.Context, id string) error
|
||||
}
|
||||
|
||||
// chatReattachPoll is how often a reattached turn re-asks its harness whether
|
||||
@@ -63,10 +64,15 @@ func (h *chatHub) detachSessions(ctx context.Context) error {
|
||||
if len(live) > 0 {
|
||||
log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live))
|
||||
}
|
||||
return h.liveStore.RecordChatLiveSessions(ctx, live)
|
||||
for _, session := range live {
|
||||
if err := h.liveStore.RememberChatLiveSession(ctx, session); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reattachSessions is the start-up half of detachSessions: each recorded chat
|
||||
// reattachSessions reads (without consuming) the recovery journal. Each chat
|
||||
// is observed again (stream, pending questions, background tasks) while its
|
||||
// harness still has work, and a turn that ended while the server was down is
|
||||
// folded in from the harness's history, so its reply still notifies and its
|
||||
@@ -75,7 +81,7 @@ func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
|
||||
if h.liveStore == nil || broker == nil {
|
||||
return
|
||||
}
|
||||
live, err := h.liveStore.TakeChatLiveSessions(ctx)
|
||||
live, err := h.liveStore.ChatLiveSessions(ctx)
|
||||
if err != nil {
|
||||
log.Printf("chat reattach: %v", err)
|
||||
return
|
||||
@@ -87,13 +93,34 @@ func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
|
||||
if !validChatSessionID(session.SessionID) {
|
||||
continue
|
||||
}
|
||||
h.mu.Lock()
|
||||
observed := h.rooms[session.SessionID] != nil
|
||||
generation := h.recoveryGeneration[session.SessionID]
|
||||
h.mu.Unlock()
|
||||
if observed {
|
||||
continue
|
||||
}
|
||||
if h.registry != nil {
|
||||
row, found, err := h.registry.OperatorSession(ctx, session.SessionID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !found || row.ClosedAt != 0 {
|
||||
h.forgetLive(ctx, session.SessionID, generation)
|
||||
continue
|
||||
}
|
||||
}
|
||||
h.reattachSession(ctx, broker, session)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) {
|
||||
id := session.SessionID
|
||||
h.mu.Lock()
|
||||
generation := h.recoveryGeneration[id]
|
||||
h.mu.Unlock()
|
||||
running := false
|
||||
turnKnown := false
|
||||
if prober, ok := broker.(chatTurnProber); ok {
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
busy, err := prober.ProbeTurn(probeCtx, id)
|
||||
@@ -104,6 +131,54 @@ func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, sessio
|
||||
busy = session.Turn
|
||||
}
|
||||
running = busy
|
||||
turnKnown = err == nil
|
||||
}
|
||||
var snapshot []protocol.ChatShell
|
||||
attentionKnown := false
|
||||
if !running {
|
||||
// A recovered task may have ended while disconnected. Only clear the
|
||||
// journal after all pending-work surfaces authoritatively report idle.
|
||||
if attention, known, shells := probeChatAttention(ctx, broker, id); known {
|
||||
snapshot = shells
|
||||
attentionKnown = true
|
||||
session.Attention = attention
|
||||
} else {
|
||||
session.Attention = true // Unknown pending work is not idle.
|
||||
}
|
||||
}
|
||||
h.mu.Lock()
|
||||
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
|
||||
if !stale && attentionKnown {
|
||||
h.replaceShellsLocked(id, snapshot)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if stale {
|
||||
return
|
||||
}
|
||||
if turnKnown && !running && !session.Attention {
|
||||
if session.Turn {
|
||||
h.foldEndedTurn(ctx, broker, session, nil)
|
||||
}
|
||||
if h.liveStore != nil {
|
||||
h.forgetLive(ctx, id, generation)
|
||||
}
|
||||
return
|
||||
}
|
||||
retained := session
|
||||
retained.Turn = running
|
||||
retained.RecordedAt = h.clock()
|
||||
if !h.rememberRecovered(ctx, retained, generation) {
|
||||
return
|
||||
}
|
||||
if running && session.Turn {
|
||||
// Seed ownership before opening the stream: replay/poll admission can
|
||||
// journal immediately and must not overwrite the recovered running turn.
|
||||
h.mu.Lock()
|
||||
turn := h.turnLocked(id)
|
||||
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
|
||||
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
if running || session.Attention {
|
||||
h.observeUnattended(id, broker)
|
||||
|
||||
@@ -56,7 +56,10 @@ type chatHub struct {
|
||||
queueStore chatPromptQueueStore
|
||||
// liveStore records the chats a stopping server leaves live on their
|
||||
// harness (chat_detach.go); nil reattaches nothing beyond the start probe.
|
||||
liveStore chatLiveStore
|
||||
liveStore chatLiveStore
|
||||
liveJournal map[string]chatRecoveryState
|
||||
recoveryGeneration map[string]uint64
|
||||
recoveryDone <-chan struct{}
|
||||
// recentOps/recentOpOrder are the bounded record of accepted prompt
|
||||
// operation ids, so a client retry is acknowledged instead of re-sent.
|
||||
recentOps map[string]chatRecentOp
|
||||
@@ -202,7 +205,11 @@ func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
|
||||
if room := h.rooms[frame.SessionID]; room != nil {
|
||||
room.shellReadback.Record(frame.Shell)
|
||||
}
|
||||
if frame.Shell != nil && frame.Shell.ID != "" {
|
||||
h.touchRecoveryLocked(frame.SessionID)
|
||||
}
|
||||
turn.applyShell(frame.Shell)
|
||||
h.rememberLiveLocked(frame.SessionID)
|
||||
return
|
||||
case protocol.ChatEventForm:
|
||||
// A form/question likewise carries no message id. Keep the pending form
|
||||
@@ -221,6 +228,7 @@ func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
|
||||
if !frame.Form.Resolved && !frame.Form.Cancelled && len(room.forms) < 8 {
|
||||
room.forms = append(room.forms, *frame.Form)
|
||||
}
|
||||
h.rememberLiveLocked(frame.SessionID)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
type chatRecoveryState struct {
|
||||
turn, attention bool
|
||||
native string
|
||||
}
|
||||
|
||||
// rememberLiveLocked journals liveness before an observer can be evicted or
|
||||
// the process can crash. A bounded replay cache is never the recovery journal.
|
||||
// Callers hold h.mu; like durable prompt admission, this small SQLite write is
|
||||
// ordered with the lifecycle change it describes.
|
||||
func (h *chatHub) rememberLiveLocked(id string) {
|
||||
if h.liveStore == nil || (h.baseCtx != nil && h.baseCtx.Err() != nil) {
|
||||
return
|
||||
}
|
||||
room := h.rooms[id]
|
||||
if room == nil {
|
||||
return
|
||||
} // Losing observation is not authoritative idle.
|
||||
turn := room.running || room.observedTurn
|
||||
native := room.nativeTurnID
|
||||
if current := h.turns[id]; current != nil && current.step != nil && current.step.Status == protocol.ChatStepStarted && !current.step.Intermediate {
|
||||
turn = true
|
||||
if native == "" {
|
||||
native = current.step.TurnID
|
||||
}
|
||||
}
|
||||
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
|
||||
next := chatRecoveryState{turn: turn, attention: attention, native: native}
|
||||
if previous, ok := h.liveJournal[id]; ok && previous == next {
|
||||
return
|
||||
}
|
||||
h.touchRecoveryLocked(id)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := h.liveStore.RememberChatLiveSession(ctx, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: native, Attention: attention, RecordedAt: h.clock()}); err != nil {
|
||||
log.Printf("chat recovery journal session=%s failed (%T)", id, err)
|
||||
return
|
||||
}
|
||||
if h.liveJournal == nil {
|
||||
h.liveJournal = map[string]chatRecoveryState{}
|
||||
}
|
||||
h.liveJournal[id] = next
|
||||
}
|
||||
|
||||
func (h *chatHub) touchRecoveryLocked(id string) {
|
||||
if h.recoveryGeneration == nil {
|
||||
h.recoveryGeneration = map[string]uint64{}
|
||||
}
|
||||
h.recoveryGeneration[id]++
|
||||
}
|
||||
|
||||
// Probe results are older than any lifecycle that arrived while they were in
|
||||
// flight. Hold the same lock as lifecycle journaling through the final delete.
|
||||
func (h *chatHub) forgetLive(ctx context.Context, id string, generation uint64) {
|
||||
if h.liveStore == nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.recoveryGeneration[id] != generation || h.rooms[id] != nil || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := h.liveStore.ForgetChatLiveSession(ctx, id); err != nil {
|
||||
log.Printf("chat recovery clear session=%s failed (%T)", id, err)
|
||||
return
|
||||
}
|
||||
delete(h.liveJournal, id)
|
||||
}
|
||||
|
||||
func (h *chatHub) rememberRecovered(ctx context.Context, v pipeline.ChatLiveSession, generation uint64) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.recoveryGeneration[v.SessionID] != generation || h.rooms[v.SessionID] != nil || ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
h.touchRecoveryLocked(v.SessionID)
|
||||
if h.liveStore != nil {
|
||||
if err := h.liveStore.RememberChatLiveSession(ctx, v); err != nil {
|
||||
log.Printf("chat reattach journal session=%s failed (%T)", v.SessionID, err)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// probeChatAttention reads only authoritative pending-work surfaces, never
|
||||
// starts a model turn or infers task completion from absence of foreground work.
|
||||
func probeChatAttention(ctx context.Context, broker chatBroker, id string) (attention, known bool, shells []protocol.ChatShell) {
|
||||
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
shells, err := broker.Shells(probe, id)
|
||||
if err != nil {
|
||||
return false, false, nil
|
||||
}
|
||||
for _, shell := range shells {
|
||||
if shell.Status == protocol.ChatShellRunning {
|
||||
return true, true, shells
|
||||
}
|
||||
}
|
||||
permissions, err := broker.Permissions(probe, id)
|
||||
if err != nil {
|
||||
return false, false, shells
|
||||
}
|
||||
for _, p := range permissions {
|
||||
if !p.Resolved {
|
||||
return true, true, shells
|
||||
}
|
||||
}
|
||||
if reader, ok := broker.(chatFormReader); ok {
|
||||
forms, err := reader.Forms(probe, id)
|
||||
if err != nil {
|
||||
return false, false, shells
|
||||
}
|
||||
for _, f := range forms {
|
||||
if !f.Resolved && !f.Cancelled {
|
||||
return true, true, shells
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, true, shells
|
||||
}
|
||||
|
||||
const chatRecoveryRetry = 30 * time.Second
|
||||
|
||||
// Only explicitly journaled but unobserved sessions are retried. Transient
|
||||
// harness unavailability must survive more than one start, without polling all
|
||||
// historical chats repeatedly.
|
||||
func (h *chatHub) retryReattachments(ctx context.Context, broker chatBroker) {
|
||||
ticker := time.NewTicker(chatRecoveryRetry)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
h.reattachSessions(ctx, broker)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recoverDiscoveredSession attaches only if the harness reports actual work.
|
||||
// Successful foreground-idle alone never rules out background tasks/questions.
|
||||
func (h *chatHub) recoverDiscoveredSession(ctx context.Context, broker chatBroker, id string) {
|
||||
h.mu.Lock()
|
||||
observed := h.rooms[id] != nil
|
||||
generation := h.recoveryGeneration[id]
|
||||
h.mu.Unlock()
|
||||
if observed || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
running := false
|
||||
if prober, ok := broker.(chatTurnProber); ok {
|
||||
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
busy, err := prober.ProbeTurn(probe, id)
|
||||
cancel()
|
||||
if err == nil {
|
||||
running = busy
|
||||
}
|
||||
}
|
||||
attention, known, shells := probeChatAttention(ctx, broker, id)
|
||||
h.mu.Lock()
|
||||
stale := h.recoveryGeneration[id] != generation || h.rooms[id] != nil
|
||||
if !stale && known {
|
||||
h.replaceShellsLocked(id, shells)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if stale {
|
||||
return
|
||||
}
|
||||
if !running && !attention {
|
||||
return
|
||||
}
|
||||
session := pipeline.ChatLiveSession{SessionID: id, Turn: running, Attention: attention, RecordedAt: h.clock()}
|
||||
if !h.rememberRecovered(ctx, session, generation) {
|
||||
return
|
||||
}
|
||||
h.reattachSession(ctx, broker, session)
|
||||
}
|
||||
@@ -343,6 +343,7 @@ func (h *chatHub) releaseRoomLocked(room *chatRoom) {
|
||||
if h.rooms[room.sessionID] != room {
|
||||
return
|
||||
}
|
||||
h.rememberLiveLocked(room.sessionID)
|
||||
h.clearCompactionLocked(room)
|
||||
delete(h.rooms, room.sessionID)
|
||||
room.cancel()
|
||||
@@ -709,6 +710,7 @@ func (h *chatHub) pushPermission(sessionID string, frame protocol.ChatPermission
|
||||
}
|
||||
}
|
||||
room.pending = append(room.pending, frame)
|
||||
h.rememberLiveLocked(sessionID)
|
||||
h.notePermissionAttentionLocked(frame)
|
||||
if len(room.pending) > chatRoomPendingLimit {
|
||||
room.pending = room.pending[len(room.pending)-chatRoomPendingLimit:]
|
||||
|
||||
@@ -85,7 +85,13 @@ func testChatShellOutputOwnedAndForeign(t *testing.T, exited bool) {
|
||||
if err := store.RecordOperatorSession(ctx, LargeChatFixtureSessionID, t.TempDir(), "owned", protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store}))
|
||||
handler := HandlerWithOptions(ctx, Options{Chat: opencode.NewChat(attached.Connection), OperatorRegistry: store}).(*runHandler)
|
||||
srv := httptest.NewServer(handler)
|
||||
select {
|
||||
case <-handler.hub.recoveryDone:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("startup recovery did not finish")
|
||||
}
|
||||
defer srv.Close()
|
||||
ws := dialShellOutputWebsocket(t, ctx, srv)
|
||||
defer ws.CloseNow()
|
||||
|
||||
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -17,11 +18,10 @@ import (
|
||||
// observation is started at server start for sessions whose harness reports a
|
||||
// running turn, so a reply that finishes after a restart still notifies.
|
||||
//
|
||||
// Resources are bounded: at most chatUnattendedLimit rooms are kept without a
|
||||
// client (rooms holding queued prompts are not counted: they were always kept
|
||||
// until delivered), a stuck turn or a background task that never reports an
|
||||
// exit is released after chatUnattendedTurnLimit, and an idle session is never
|
||||
// observed.
|
||||
// Idle/stale foreground observers are bounded by chatUnattendedLimit and
|
||||
// chatUnattendedTurnLimit. Known running background work (like queued prompts)
|
||||
// is not an idle lease: it remains attached until actual completion or explicit
|
||||
// stop. Its durable journal is independent of bounded replay/room retention.
|
||||
|
||||
// chatUnattendedLimit bounds the rooms kept observing with no client attached
|
||||
// and no queued prompt.
|
||||
@@ -73,10 +73,8 @@ func (h *chatHub) keepUnattendedLocked(room *chatRoom, busy, hold bool) bool {
|
||||
if room.running || room.observedTurn || busy || len(room.pending) > 0 || len(room.forms) > 0 {
|
||||
return true
|
||||
}
|
||||
// Running background tasks keep the stream observed so their exits are
|
||||
// seen and the chat list's indicator clears. This is bounded like a turn:
|
||||
// the unattended room limit, the chatUnattendedTurnLimit expiry, and the
|
||||
// harness itself (a Claude Code process that exits ends its tasks).
|
||||
// Running background tasks keep the stream observed so their actual exits
|
||||
// are seen and the chat list's indicator clears, without a fixed age cap.
|
||||
if h.backgroundTasksLocked(room.sessionID) > 0 {
|
||||
return true
|
||||
}
|
||||
@@ -108,7 +106,7 @@ func (h *chatHub) leaveUnattendedLocked(room *chatRoom) {
|
||||
h.releaseRoomLocked(room)
|
||||
return
|
||||
}
|
||||
if !room.watching && h.unattendedCountLocked(room) >= chatUnattendedLimit {
|
||||
if !room.watching && h.backgroundTasksLocked(room.sessionID) == 0 && h.unattendedCountLocked(room) >= chatUnattendedLimit {
|
||||
log.Printf("chat session %s: %d chats already observed with no client; not keeping this one", room.sessionID, chatUnattendedLimit)
|
||||
h.releaseRoomLocked(room)
|
||||
return
|
||||
@@ -172,6 +170,11 @@ func (h *chatHub) reapUnattended(room *chatRoom) bool {
|
||||
room.unattendedSince = h.clock()
|
||||
return false
|
||||
}
|
||||
// A running task is actual work, not an idle-observer lease. Its terminal
|
||||
// lifecycle/readback, or explicit operator stop, releases it normally.
|
||||
if h.backgroundTasksLocked(room.sessionID) > 0 {
|
||||
return false
|
||||
}
|
||||
expired := h.clock().Sub(room.unattendedSince) >= h.unattended.turnLimit
|
||||
if !expired && h.keepUnattendedLocked(room, busy, true) {
|
||||
return false
|
||||
@@ -199,45 +202,56 @@ func (h *chatHub) attendedLocked(room *chatRoom) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// observeRunningTurns is the start-up half: each recently used open operator
|
||||
// session whose harness reports a turn in flight gets an unattended room
|
||||
// observing its stream, so the reply (or a question) still notifies. It is
|
||||
// best effort per harness: OpenCode and Codex keep running across a Pipelines
|
||||
// restart and report it; Claude Code runs as a child of the server and so has
|
||||
// no turn left after a restart.
|
||||
// observeRunningTurns discovers live foreground or background work even when
|
||||
// no observer survived long enough to journal it (including a first upgrade or
|
||||
// an abrupt old-server exit). Every open registered session is considered; old
|
||||
// metadata is not evidence that a long-running task ended. Work is bounded to
|
||||
// batches of 32 with four concurrent readback probes, never prompt/create/resume.
|
||||
func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
|
||||
prober, ok := broker.(chatTurnProber)
|
||||
if !ok || h.registry == nil {
|
||||
if h.registry == nil {
|
||||
return
|
||||
}
|
||||
sessions, err := h.registry.OperatorSessions(ctx)
|
||||
if err != nil {
|
||||
log.Printf("chat turn recovery: %v", err)
|
||||
log.Printf("chat recovery registry: %v", err)
|
||||
return
|
||||
}
|
||||
cutoff := h.clock().Add(-h.unattended.window).UnixMilli()
|
||||
probed := 0
|
||||
ids := make([]string, 0, len(sessions))
|
||||
for _, session := range sessions {
|
||||
if session.ClosedAt == 0 && validChatSessionID(session.ID) {
|
||||
ids = append(ids, session.ID)
|
||||
}
|
||||
}
|
||||
for start := 0; start < len(ids); start += chatRecoverSessionLimit {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if session.ClosedAt != 0 || session.UpdatedAt < cutoff || !validChatSessionID(session.ID) {
|
||||
continue
|
||||
end := start + chatRecoverSessionLimit
|
||||
if end > len(ids) {
|
||||
end = len(ids)
|
||||
}
|
||||
if probed >= chatRecoverSessionLimit {
|
||||
break
|
||||
jobs := make(chan string)
|
||||
var workers sync.WaitGroup
|
||||
for i := 0; i < 4; i++ {
|
||||
workers.Add(1)
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for id := range jobs {
|
||||
h.recoverDiscoveredSession(ctx, broker, id)
|
||||
}
|
||||
}()
|
||||
}
|
||||
probed++
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
running, err := prober.ProbeTurn(probeCtx, session.ID)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("chat turn recovery probe session=%s failed (%T)", session.ID, err)
|
||||
continue
|
||||
}
|
||||
if running {
|
||||
h.observeUnattended(session.ID, broker)
|
||||
for _, id := range ids[start:end] {
|
||||
select {
|
||||
case jobs <- id:
|
||||
case <-ctx.Done():
|
||||
close(jobs)
|
||||
workers.Wait()
|
||||
return
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
workers.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,7 +259,7 @@ func (h *chatHub) observeRunningTurns(ctx context.Context, broker chatBroker) {
|
||||
// observes its stream until the turn the harness reported has ended.
|
||||
func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
|
||||
h.mu.Lock()
|
||||
if h.rooms[sessionID] == nil && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
|
||||
if h.rooms[sessionID] == nil && h.backgroundTasksLocked(sessionID) == 0 && h.unattendedCountLocked(nil) >= chatUnattendedLimit {
|
||||
h.mu.Unlock()
|
||||
log.Printf("chat session %s: %d chats already observed with no client; not observing", sessionID, chatUnattendedLimit)
|
||||
return
|
||||
|
||||
@@ -279,9 +279,9 @@ func TestChatRunningTurnObservedAfterRestart(t *testing.T) {
|
||||
waitFor(t, "observer torn down", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
}
|
||||
|
||||
// A recovered session whose stream never reports the turn is released after
|
||||
// the settle window rather than observed forever.
|
||||
func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) {
|
||||
// Authoritative busy survives the settle window even without a replayed start;
|
||||
// a later authoritative idle result releases it.
|
||||
func TestChatRecoveredBusySessionWaitsForAuthoritativeIdle(t *testing.T) {
|
||||
fastUnattended(t)
|
||||
t.Setenv("PIPELINES_NTFY_URL", "")
|
||||
store := unattendedStore(t, true)
|
||||
@@ -289,7 +289,14 @@ func TestChatRecoveredSessionWithoutTurnIsReleased(t *testing.T) {
|
||||
broker.probe["ses_1"] = true
|
||||
startQueueServer(t, broker, store)
|
||||
waitFor(t, "recovered stream", func() bool { active, _ := broker.streams(); return active == 1 })
|
||||
waitFor(t, "released after settle", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
time.Sleep(2 * chatUnattendedRecoverSettle)
|
||||
if active, _ := broker.streams(); active != 1 {
|
||||
t.Fatal("authoritative busy session lost observer")
|
||||
}
|
||||
broker.smu.Lock()
|
||||
broker.probe["ses_1"] = false
|
||||
broker.smu.Unlock()
|
||||
waitFor(t, "released after authoritative idle", func() bool { active, _ := broker.streams(); return active == 0 })
|
||||
}
|
||||
|
||||
// A turn that never reports its end does not hold an observer forever.
|
||||
|
||||
@@ -134,9 +134,6 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
// session; resume delivering any a previous process left queued.
|
||||
hub.queueStore = stores[0]
|
||||
hub.liveStore = stores[0]
|
||||
if options.Chat != nil {
|
||||
go hub.resumeQueued(ctx, options.Chat)
|
||||
}
|
||||
}
|
||||
acks := newAttentionAckHub()
|
||||
var ackStore attentionAckStore
|
||||
@@ -500,9 +497,15 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
// that ended while it was down), then keep observing any other chat
|
||||
// whose turn is still running, so a reply or question still notifies
|
||||
// with no client attached.
|
||||
done := make(chan struct{})
|
||||
hub.recoveryDone = done
|
||||
go func() {
|
||||
hub.reattachSessions(ctx, options.Chat)
|
||||
// Restore live turn ownership before queue rooms can start runners.
|
||||
hub.resumeQueued(ctx, options.Chat)
|
||||
hub.observeRunningTurns(ctx, options.Chat)
|
||||
close(done)
|
||||
hub.retryReattachments(ctx, options.Chat)
|
||||
}()
|
||||
}
|
||||
if runs != nil {
|
||||
|
||||
Reference in New Issue
Block a user