package server import ( "context" "database/sql" "errors" "fmt" "sync" "sync/atomic" "testing" "time" "github.com/coder/websocket/wsjson" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" ) type recoveryTasks struct { *restartChat mu sync.Mutex tasks map[string][]protocol.ChatShell unavailable bool historyReads int creates int } func newRecoveryTasks() *recoveryTasks { return &recoveryTasks{restartChat: newRestartChat(), tasks: map[string][]protocol.ChatShell{}} } func (b *recoveryTasks) Shells(_ context.Context, id string) ([]protocol.ChatShell, error) { b.mu.Lock() defer b.mu.Unlock() if b.unavailable { return nil, errors.New("temporary read failure") } return append([]protocol.ChatShell(nil), b.tasks[id]...), nil } func (b *recoveryTasks) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) { b.mu.Lock() defer b.mu.Unlock() b.historyReads++ return protocol.ChatHistoryPage{}, nil } func (b *recoveryTasks) Create(context.Context, string, string) (protocol.ChatSession, error) { b.mu.Lock() defer b.mu.Unlock() b.creates++ return protocol.ChatSession{}, errors.New("recovery must not create a session") } func task(id string) protocol.ChatShell { return protocol.ChatShell{ID: id, Status: protocol.ChatShellRunning, Command: "isolated background task", StartedAt: 1234} } func waitRecoveredTask(t *testing.T, s *chatRestartServer, id, taskID string) { t.Helper() waitFor(t, "background reattached with same identity", func() bool { h := s.handler.hub h.mu.Lock() room := h.rooms[id] attached := room != nil && room.streaming h.mu.Unlock() if !attached { return false } got := h.shellSnapshot(id) return len(got) == 1 && got[0].ID == taskID && got[0].StartedAt == 1234 }) } func TestBackgroundRecoverySurvivesRoomEvictionAndUngracefulStop(t *testing.T) { fastReattach(t) path := restartChatDB(t) first := newRecoveryTasks() first.tasks["ses_1"] = []protocol.ChatShell{task("job_same")} s := startChatRestartServer(t, first, path) waitRecoveredTask(t, s, "ses_1", "job_same") h := s.handler.hub h.mu.Lock() h.releaseRoomLocked(h.rooms["ses_1"]) h.mu.Unlock() // No DetachSessions: model a crash after the observer was released. s.stop() second := newRecoveryTasks() second.tasks["ses_1"] = []protocol.ChatShell{task("job_same")} next := startChatRestartServer(t, second, path) waitRecoveredTask(t, next, "ses_1", "job_same") second.mu.Lock() defer second.mu.Unlock() if second.historyReads != 0 || second.creates != 0 || second.promptCount() != 0 { t.Fatal("recovery started/read a conversation instead of reattaching tasks") } } func TestBackgroundRecoveryRetainsJournalAcrossFailedReattachment(t *testing.T) { fastReattach(t) path := restartChatDB(t) store := openQueueStore(t, path) if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}); err != nil { t.Fatal(err) } broken := newRecoveryTasks() broken.unavailable = true first := startChatRestartServer(t, broken, path) select { case <-first.handler.hub.recoveryDone: case <-first.ctx.Done(): t.Fatal("recovery did not settle") } records, err := store.ChatLiveSessions(context.Background()) if err != nil || len(records) != 1 { t.Fatalf("failed reconnect consumed journal: %+v %v", records, err) } first.stop() repaired := newRecoveryTasks() repaired.tasks["ses_1"] = []protocol.ChatShell{task("job_kept")} next := startChatRestartServer(t, repaired, path) waitRecoveredTask(t, next, "ses_1", "job_kept") } func TestBackgroundRecoveryFindsOldSessionsBeyondFormerRecentLimit(t *testing.T) { fastReattach(t) path := restartChatDB(t) store := openQueueStore(t, path) for i := 0; i < chatRecoverSessionLimit+5; i++ { if err := store.RecordOperatorSession(context.Background(), fmt.Sprintf("ses_new_%02d", i), t.TempDir(), "idle", protocol.ChatSessionTagOperator); err != nil { t.Fatal(err) } } db, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } defer db.Close() if _, err := db.Exec(`UPDATE operator_sessions SET last_used_at=? WHERE session_id='ses_1'`, time.Now().Add(-7*24*time.Hour).UnixMilli()); err != nil { t.Fatal(err) } broker := newRecoveryTasks() broker.tasks["ses_1"] = []protocol.ChatShell{task("job_old")} s := startChatRestartServer(t, broker, path) waitRecoveredTask(t, s, "ses_1", "job_old") broker.mu.Lock() defer broker.mu.Unlock() if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 { t.Fatal("discovery resumed or read chat history") } } func TestBackgroundRecoveryKeepsLiveObserversBeyondIdleCapAndTTL(t *testing.T) { fastReattach(t) path := restartChatDB(t) store := openQueueStore(t, path) broker := newRecoveryTasks() count := chatUnattendedLimit + 3 for i := 0; i < count; i++ { id := fmt.Sprintf("ses_busy_%02d", i) if err := store.RecordOperatorSession(context.Background(), id, t.TempDir(), "background", protocol.ChatSessionTagOperator); err != nil { t.Fatal(err) } broker.tasks[id] = []protocol.ChatShell{task("job_" + id)} } s := startChatRestartServer(t, broker, path) waitFor(t, "all background streams restored despite idle cap", func() bool { active, _ := broker.streams(); return active == count }) h := s.handler.hub h.mu.Lock() rooms := make([]*chatRoom, 0, len(h.rooms)) for _, room := range h.rooms { room.unattendedSince = h.clock().Add(-7 * time.Hour) rooms = append(rooms, room) } h.mu.Unlock() for _, room := range rooms { if h.reapUnattended(room) { t.Fatal("live task observer expired") } } if active, _ := broker.streams(); active != count { t.Fatalf("lost background streams: %d", active) } } // A readback captured before a newer task event must not erase that task's // durable recovery membership, even if its observer has since been released. type staleRecoveryTasks struct { *recoveryTasks entered chan struct{} release chan struct{} } func (b *staleRecoveryTasks) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) { close(b.entered) select { case <-b.release: return nil, nil case <-ctx.Done(): return nil, ctx.Err() } } func TestBackgroundRecoveryStaleIdleCannotForgetNewTask(t *testing.T) { ctx := context.Background() store := openQueueStore(t, restartChatDB(t)) old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true} if err := store.RememberChatLiveSession(ctx, old); err != nil { t.Fatal(err) } h := newChatHub() h.liveStore = store broker := &staleRecoveryTasks{recoveryTasks: newRecoveryTasks(), entered: make(chan struct{}), release: make(chan struct{})} done := make(chan struct{}) go func() { h.reattachSession(ctx, broker, old); close(done) }() <-broker.entered h.mu.Lock() room := h.roomLocked("ses_1") newest := task("new_job") h.applyLocked(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &newest}) h.releaseRoomLocked(room) h.mu.Unlock() close(broker.release) <-done rows, err := store.ChatLiveSessions(ctx) if err != nil { t.Fatal(err) } if len(rows) != 1 || !rows[0].Attention { t.Fatalf("stale idle erased new task journal: %+v", rows) } } func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T) { ctx := context.Background() store := openQueueStore(t, restartChatDB(t)) old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true} if err := store.RememberChatLiveSession(ctx, old); err != nil { t.Fatal(err) } h := newChatHub() h.liveStore = store broker := newRecoveryTasks() h.reattachSession(ctx, broker, old) rows, err := store.ChatLiveSessions(ctx) if err != nil { t.Fatal(err) } if len(rows) != 0 { t.Fatalf("authoritative idle left stale journal: %+v", rows) } if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 { t.Fatal("idle recovery started or read a conversation") } } type blockedRecoveryProbe struct { *restartChat shellReads atomic.Int32 once sync.Once entered chan struct{} release chan struct{} } func (b *blockedRecoveryProbe) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) { b.shellReads.Add(1) return b.restartChat.Shells(ctx, id) } func (b *blockedRecoveryProbe) ProbeTurn(ctx context.Context, id string) (bool, error) { b.once.Do(func() { close(b.entered) select { case <-b.release: case <-ctx.Done(): } }) return b.restartChat.ProbeTurn(ctx, id) } func TestChatImmediateSubscribeWaitsForRecoveredTurnOwnership(t *testing.T) { fastReattach(t) path := restartChatDB(t) startTurnThenRestart(t, path, true) broker := &blockedRecoveryProbe{restartChat: newRestartChat(), entered: make(chan struct{}), release: make(chan struct{})} broker.setProbe(true) server := startChatRestartServer(t, broker, path) <-broker.entered conn := dialChatConn(t, server.ctx, server.srv) if err := wsjson.Write(server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "early", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { t.Fatal(err) } readbackConn := dialChatConn(t, server.ctx, server.srv) if err := wsjson.Write(server.ctx, readbackConn, protocol.Message{Type: protocol.ChatShells, ID: "early-shells", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { t.Fatal(err) } // The request is on the live socket while reconciliation is blocked. It // must not create a room that makes recovery skip the durable running turn. time.Sleep(2 * chatQueueRecoverySettle) server.handler.hub.mu.Lock() room := server.handler.hub.rooms["ses_1"] server.handler.hub.mu.Unlock() if room != nil { t.Fatal("early subscribe admitted a room before recovery") } if broker.shellReads.Load() != 0 { t.Fatal("early shell readback bypassed recovery admission") } close(broker.release) for { var ack protocol.Message if err := readTestFrame(server.ctx, conn, &ack); err != nil { t.Fatal(err) } if ack.Type == protocol.ChatSubscribe && ack.ID == "early" { break } if ack.Type == protocol.Error { t.Fatalf("subscribe error: %s", ack.Error) } } time.Sleep(2 * chatQueueRecoverySettle) if broker.promptCount() != 0 { t.Fatalf("queued prompt delivered into recovered running turn: %v", broker.promptTexts()) } broker.setProbe(false) waitFor(t, "exactly one queued prompt delivered after recovered turn", func() bool { return broker.promptCount() == 1 }) if got := broker.promptTexts(); len(got) != 1 || got[0] != "then ship it" { t.Fatalf("wrong or duplicate recovered prompts: %v", got) } } type failFirstRecoveryStream struct { *restartChat attempts atomic.Int32 } func (b *failFirstRecoveryStream) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error { if b.attempts.Add(1) == 1 { return errors.New("temporary initial stream failure") } return b.restartChat.Stream(ctx, id, ready, emit) } func TestChatRecoveredForegroundRetriesInitialStreamFailure(t *testing.T) { fastReattach(t) path := restartChatDB(t) store := openQueueStore(t, path) if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Turn: true, NativeTurnID: "turn_original"}); err != nil { t.Fatal(err) } broker := &failFirstRecoveryStream{restartChat: newRestartChat()} broker.setProbe(true) server := startChatRestartServer(t, broker, path) waitFor(t, "foreground stream recovered without client", func() bool { active, _ := broker.streams(); return active == 1 && broker.attempts.Load() >= 2 }) h := server.handler.hub h.mu.Lock() room := h.rooms["ses_1"] owned := room != nil && room.observedTurn && room.nativeTurnID == "turn_original" h.mu.Unlock() if !owned { t.Fatal("stream retry lost recovered foreground ownership") } if broker.promptCount() != 0 { t.Fatal("stream retry prompted model") } broker.setProbe(false) waitFor(t, "recovered foreground authoritative completion", func() bool { h.mu.Lock(); defer h.mu.Unlock(); return h.rooms["ses_1"] == nil }) }