Files
pipelines/internal/server/chat_background_recovery_test.go

362 lines
12 KiB
Go

package server
import (
"context"
"database/sql"
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/coder/websocket/wsjson"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type recoveryTasks struct {
*restartChat
mu sync.Mutex
tasks map[string][]protocol.ChatShell
unavailable bool
historyReads int
creates int
}
func newRecoveryTasks() *recoveryTasks {
return &recoveryTasks{restartChat: newRestartChat(), tasks: map[string][]protocol.ChatShell{}}
}
func (b *recoveryTasks) Shells(_ context.Context, id string) ([]protocol.ChatShell, error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.unavailable {
return nil, errors.New("temporary read failure")
}
return append([]protocol.ChatShell(nil), b.tasks[id]...), nil
}
func (b *recoveryTasks) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.historyReads++
return protocol.ChatHistoryPage{}, nil
}
func (b *recoveryTasks) Create(context.Context, string, string) (protocol.ChatSession, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.creates++
return protocol.ChatSession{}, errors.New("recovery must not create a session")
}
func task(id string) protocol.ChatShell {
return protocol.ChatShell{ID: id, Status: protocol.ChatShellRunning, Command: "isolated background task", StartedAt: 1234}
}
func waitRecoveredTask(t *testing.T, s *chatRestartServer, id, taskID string) {
t.Helper()
waitFor(t, "background reattached with same identity", func() bool {
h := s.handler.hub
h.mu.Lock()
room := h.rooms[id]
attached := room != nil && room.streaming
h.mu.Unlock()
if !attached {
return false
}
got := h.shellSnapshot(id)
return len(got) == 1 && got[0].ID == taskID && got[0].StartedAt == 1234
})
}
func TestBackgroundRecoverySurvivesRoomEvictionAndUngracefulStop(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
first := newRecoveryTasks()
first.tasks["ses_1"] = []protocol.ChatShell{task("job_same")}
s := startChatRestartServer(t, first, path)
waitRecoveredTask(t, s, "ses_1", "job_same")
h := s.handler.hub
h.mu.Lock()
h.releaseRoomLocked(h.rooms["ses_1"])
h.mu.Unlock()
// No DetachSessions: model a crash after the observer was released.
s.stop()
second := newRecoveryTasks()
second.tasks["ses_1"] = []protocol.ChatShell{task("job_same")}
next := startChatRestartServer(t, second, path)
waitRecoveredTask(t, next, "ses_1", "job_same")
second.mu.Lock()
defer second.mu.Unlock()
if second.historyReads != 0 || second.creates != 0 || second.promptCount() != 0 {
t.Fatal("recovery started/read a conversation instead of reattaching tasks")
}
}
func TestBackgroundRecoveryRetainsJournalAcrossFailedReattachment(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
store := openQueueStore(t, path)
if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}); err != nil {
t.Fatal(err)
}
broken := newRecoveryTasks()
broken.unavailable = true
first := startChatRestartServer(t, broken, path)
select {
case <-first.handler.hub.recoveryDone:
case <-first.ctx.Done():
t.Fatal("recovery did not settle")
}
records, err := store.ChatLiveSessions(context.Background())
if err != nil || len(records) != 1 {
t.Fatalf("failed reconnect consumed journal: %+v %v", records, err)
}
first.stop()
repaired := newRecoveryTasks()
repaired.tasks["ses_1"] = []protocol.ChatShell{task("job_kept")}
next := startChatRestartServer(t, repaired, path)
waitRecoveredTask(t, next, "ses_1", "job_kept")
}
func TestBackgroundRecoveryFindsOldSessionsBeyondFormerRecentLimit(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
store := openQueueStore(t, path)
for i := 0; i < chatRecoverSessionLimit+5; i++ {
if err := store.RecordOperatorSession(context.Background(), fmt.Sprintf("ses_new_%02d", i), t.TempDir(), "idle", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
}
db, err := sql.Open("sqlite", path)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Exec(`UPDATE operator_sessions SET last_used_at=? WHERE session_id='ses_1'`, time.Now().Add(-7*24*time.Hour).UnixMilli()); err != nil {
t.Fatal(err)
}
broker := newRecoveryTasks()
broker.tasks["ses_1"] = []protocol.ChatShell{task("job_old")}
s := startChatRestartServer(t, broker, path)
waitRecoveredTask(t, s, "ses_1", "job_old")
broker.mu.Lock()
defer broker.mu.Unlock()
if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 {
t.Fatal("discovery resumed or read chat history")
}
}
func TestBackgroundRecoveryKeepsLiveObserversBeyondIdleCapAndTTL(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
store := openQueueStore(t, path)
broker := newRecoveryTasks()
count := chatUnattendedLimit + 3
for i := 0; i < count; i++ {
id := fmt.Sprintf("ses_busy_%02d", i)
if err := store.RecordOperatorSession(context.Background(), id, t.TempDir(), "background", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
broker.tasks[id] = []protocol.ChatShell{task("job_" + id)}
}
s := startChatRestartServer(t, broker, path)
waitFor(t, "all background streams restored despite idle cap", func() bool { active, _ := broker.streams(); return active == count })
h := s.handler.hub
h.mu.Lock()
rooms := make([]*chatRoom, 0, len(h.rooms))
for _, room := range h.rooms {
room.unattendedSince = h.clock().Add(-7 * time.Hour)
rooms = append(rooms, room)
}
h.mu.Unlock()
for _, room := range rooms {
if h.reapUnattended(room) {
t.Fatal("live task observer expired")
}
}
if active, _ := broker.streams(); active != count {
t.Fatalf("lost background streams: %d", active)
}
}
// A readback captured before a newer task event must not erase that task's
// durable recovery membership, even if its observer has since been released.
type staleRecoveryTasks struct {
*recoveryTasks
entered chan struct{}
release chan struct{}
}
func (b *staleRecoveryTasks) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
close(b.entered)
select {
case <-b.release:
return nil, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func TestBackgroundRecoveryStaleIdleCannotForgetNewTask(t *testing.T) {
ctx := context.Background()
store := openQueueStore(t, restartChatDB(t))
old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}
if err := store.RememberChatLiveSession(ctx, old); err != nil {
t.Fatal(err)
}
h := newChatHub()
h.liveStore = store
broker := &staleRecoveryTasks{recoveryTasks: newRecoveryTasks(), entered: make(chan struct{}), release: make(chan struct{})}
done := make(chan struct{})
go func() { h.reattachSession(ctx, broker, old); close(done) }()
<-broker.entered
h.mu.Lock()
room := h.roomLocked("ses_1")
newest := task("new_job")
h.applyLocked(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &newest})
h.releaseRoomLocked(room)
h.mu.Unlock()
close(broker.release)
<-done
rows, err := store.ChatLiveSessions(ctx)
if err != nil {
t.Fatal(err)
}
if len(rows) != 1 || !rows[0].Attention {
t.Fatalf("stale idle erased new task journal: %+v", rows)
}
}
func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T) {
ctx := context.Background()
store := openQueueStore(t, restartChatDB(t))
old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}
if err := store.RememberChatLiveSession(ctx, old); err != nil {
t.Fatal(err)
}
h := newChatHub()
h.liveStore = store
broker := newRecoveryTasks()
h.reattachSession(ctx, broker, old)
rows, err := store.ChatLiveSessions(ctx)
if err != nil {
t.Fatal(err)
}
if len(rows) != 0 {
t.Fatalf("authoritative idle left stale journal: %+v", rows)
}
if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 {
t.Fatal("idle recovery started or read a conversation")
}
}
type blockedRecoveryProbe struct {
*restartChat
shellReads atomic.Int32
once sync.Once
entered chan struct{}
release chan struct{}
}
func (b *blockedRecoveryProbe) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
b.shellReads.Add(1)
return b.restartChat.Shells(ctx, id)
}
func (b *blockedRecoveryProbe) ProbeTurn(ctx context.Context, id string) (bool, error) {
b.once.Do(func() {
close(b.entered)
select {
case <-b.release:
case <-ctx.Done():
}
})
return b.restartChat.ProbeTurn(ctx, id)
}
func TestChatImmediateSubscribeWaitsForRecoveredTurnOwnership(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
startTurnThenRestart(t, path, true)
broker := &blockedRecoveryProbe{restartChat: newRestartChat(), entered: make(chan struct{}), release: make(chan struct{})}
broker.setProbe(true)
server := startChatRestartServer(t, broker, path)
<-broker.entered
conn := dialChatConn(t, server.ctx, server.srv)
if err := wsjson.Write(server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "early", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
t.Fatal(err)
}
readbackConn := dialChatConn(t, server.ctx, server.srv)
if err := wsjson.Write(server.ctx, readbackConn, protocol.Message{Type: protocol.ChatShells, ID: "early-shells", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
t.Fatal(err)
}
// The request is on the live socket while reconciliation is blocked. It
// must not create a room that makes recovery skip the durable running turn.
time.Sleep(2 * chatQueueRecoverySettle)
server.handler.hub.mu.Lock()
room := server.handler.hub.rooms["ses_1"]
server.handler.hub.mu.Unlock()
if room != nil {
t.Fatal("early subscribe admitted a room before recovery")
}
if broker.shellReads.Load() != 0 {
t.Fatal("early shell readback bypassed recovery admission")
}
close(broker.release)
for {
var ack protocol.Message
if err := readTestFrame(server.ctx, conn, &ack); err != nil {
t.Fatal(err)
}
if ack.Type == protocol.ChatSubscribe && ack.ID == "early" {
break
}
if ack.Type == protocol.Error {
t.Fatalf("subscribe error: %s", ack.Error)
}
}
time.Sleep(2 * chatQueueRecoverySettle)
if broker.promptCount() != 0 {
t.Fatalf("queued prompt delivered into recovered running turn: %v", broker.promptTexts())
}
broker.setProbe(false)
waitFor(t, "exactly one queued prompt delivered after recovered turn", func() bool { return broker.promptCount() == 1 })
if got := broker.promptTexts(); len(got) != 1 || got[0] != "then ship it" {
t.Fatalf("wrong or duplicate recovered prompts: %v", got)
}
}
type failFirstRecoveryStream struct {
*restartChat
attempts atomic.Int32
}
func (b *failFirstRecoveryStream) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
if b.attempts.Add(1) == 1 {
return errors.New("temporary initial stream failure")
}
return b.restartChat.Stream(ctx, id, ready, emit)
}
func TestChatRecoveredForegroundRetriesInitialStreamFailure(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
store := openQueueStore(t, path)
if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Turn: true, NativeTurnID: "turn_original"}); err != nil {
t.Fatal(err)
}
broker := &failFirstRecoveryStream{restartChat: newRestartChat()}
broker.setProbe(true)
server := startChatRestartServer(t, broker, path)
waitFor(t, "foreground stream recovered without client", func() bool { active, _ := broker.streams(); return active == 1 && broker.attempts.Load() >= 2 })
h := server.handler.hub
h.mu.Lock()
room := h.rooms["ses_1"]
owned := room != nil && room.observedTurn && room.nativeTurnID == "turn_original"
h.mu.Unlock()
if !owned {
t.Fatal("stream retry lost recovered foreground ownership")
}
if broker.promptCount() != 0 {
t.Fatal("stream retry prompted model")
}
broker.setProbe(false)
waitFor(t, "recovered foreground authoritative completion", func() bool { h.mu.Lock(); defer h.mu.Unlock(); return h.rooms["ses_1"] == nil })
}