362 lines
12 KiB
Go
362 lines
12 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/coder/websocket/wsjson"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
type recoveryTasks struct {
|
|
*restartChat
|
|
mu sync.Mutex
|
|
tasks map[string][]protocol.ChatShell
|
|
unavailable bool
|
|
historyReads int
|
|
creates int
|
|
}
|
|
|
|
func newRecoveryTasks() *recoveryTasks {
|
|
return &recoveryTasks{restartChat: newRestartChat(), tasks: map[string][]protocol.ChatShell{}}
|
|
}
|
|
func (b *recoveryTasks) Shells(_ context.Context, id string) ([]protocol.ChatShell, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
if b.unavailable {
|
|
return nil, errors.New("temporary read failure")
|
|
}
|
|
return append([]protocol.ChatShell(nil), b.tasks[id]...), nil
|
|
}
|
|
func (b *recoveryTasks) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.historyReads++
|
|
return protocol.ChatHistoryPage{}, nil
|
|
}
|
|
func (b *recoveryTasks) Create(context.Context, string, string) (protocol.ChatSession, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.creates++
|
|
return protocol.ChatSession{}, errors.New("recovery must not create a session")
|
|
}
|
|
func task(id string) protocol.ChatShell {
|
|
return protocol.ChatShell{ID: id, Status: protocol.ChatShellRunning, Command: "isolated background task", StartedAt: 1234}
|
|
}
|
|
func waitRecoveredTask(t *testing.T, s *chatRestartServer, id, taskID string) {
|
|
t.Helper()
|
|
waitFor(t, "background reattached with same identity", func() bool {
|
|
h := s.handler.hub
|
|
h.mu.Lock()
|
|
room := h.rooms[id]
|
|
attached := room != nil && room.streaming
|
|
h.mu.Unlock()
|
|
if !attached {
|
|
return false
|
|
}
|
|
got := h.shellSnapshot(id)
|
|
return len(got) == 1 && got[0].ID == taskID && got[0].StartedAt == 1234
|
|
})
|
|
}
|
|
|
|
func TestBackgroundRecoverySurvivesRoomEvictionAndUngracefulStop(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
first := newRecoveryTasks()
|
|
first.tasks["ses_1"] = []protocol.ChatShell{task("job_same")}
|
|
s := startChatRestartServer(t, first, path)
|
|
waitRecoveredTask(t, s, "ses_1", "job_same")
|
|
h := s.handler.hub
|
|
h.mu.Lock()
|
|
h.releaseRoomLocked(h.rooms["ses_1"])
|
|
h.mu.Unlock()
|
|
// No DetachSessions: model a crash after the observer was released.
|
|
s.stop()
|
|
second := newRecoveryTasks()
|
|
second.tasks["ses_1"] = []protocol.ChatShell{task("job_same")}
|
|
next := startChatRestartServer(t, second, path)
|
|
waitRecoveredTask(t, next, "ses_1", "job_same")
|
|
second.mu.Lock()
|
|
defer second.mu.Unlock()
|
|
if second.historyReads != 0 || second.creates != 0 || second.promptCount() != 0 {
|
|
t.Fatal("recovery started/read a conversation instead of reattaching tasks")
|
|
}
|
|
}
|
|
|
|
func TestBackgroundRecoveryRetainsJournalAcrossFailedReattachment(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
store := openQueueStore(t, path)
|
|
if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broken := newRecoveryTasks()
|
|
broken.unavailable = true
|
|
first := startChatRestartServer(t, broken, path)
|
|
select {
|
|
case <-first.handler.hub.recoveryDone:
|
|
case <-first.ctx.Done():
|
|
t.Fatal("recovery did not settle")
|
|
}
|
|
records, err := store.ChatLiveSessions(context.Background())
|
|
if err != nil || len(records) != 1 {
|
|
t.Fatalf("failed reconnect consumed journal: %+v %v", records, err)
|
|
}
|
|
first.stop()
|
|
repaired := newRecoveryTasks()
|
|
repaired.tasks["ses_1"] = []protocol.ChatShell{task("job_kept")}
|
|
next := startChatRestartServer(t, repaired, path)
|
|
waitRecoveredTask(t, next, "ses_1", "job_kept")
|
|
}
|
|
|
|
func TestBackgroundRecoveryFindsOldSessionsBeyondFormerRecentLimit(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
store := openQueueStore(t, path)
|
|
for i := 0; i < chatRecoverSessionLimit+5; i++ {
|
|
if err := store.RecordOperatorSession(context.Background(), fmt.Sprintf("ses_new_%02d", i), t.TempDir(), "idle", protocol.ChatSessionTagOperator); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.Exec(`UPDATE operator_sessions SET last_used_at=? WHERE session_id='ses_1'`, time.Now().Add(-7*24*time.Hour).UnixMilli()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker := newRecoveryTasks()
|
|
broker.tasks["ses_1"] = []protocol.ChatShell{task("job_old")}
|
|
s := startChatRestartServer(t, broker, path)
|
|
waitRecoveredTask(t, s, "ses_1", "job_old")
|
|
broker.mu.Lock()
|
|
defer broker.mu.Unlock()
|
|
if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 {
|
|
t.Fatal("discovery resumed or read chat history")
|
|
}
|
|
}
|
|
|
|
func TestBackgroundRecoveryKeepsLiveObserversBeyondIdleCapAndTTL(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
store := openQueueStore(t, path)
|
|
broker := newRecoveryTasks()
|
|
count := chatUnattendedLimit + 3
|
|
for i := 0; i < count; i++ {
|
|
id := fmt.Sprintf("ses_busy_%02d", i)
|
|
if err := store.RecordOperatorSession(context.Background(), id, t.TempDir(), "background", protocol.ChatSessionTagOperator); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker.tasks[id] = []protocol.ChatShell{task("job_" + id)}
|
|
}
|
|
s := startChatRestartServer(t, broker, path)
|
|
waitFor(t, "all background streams restored despite idle cap", func() bool { active, _ := broker.streams(); return active == count })
|
|
h := s.handler.hub
|
|
h.mu.Lock()
|
|
rooms := make([]*chatRoom, 0, len(h.rooms))
|
|
for _, room := range h.rooms {
|
|
room.unattendedSince = h.clock().Add(-7 * time.Hour)
|
|
rooms = append(rooms, room)
|
|
}
|
|
h.mu.Unlock()
|
|
for _, room := range rooms {
|
|
if h.reapUnattended(room) {
|
|
t.Fatal("live task observer expired")
|
|
}
|
|
}
|
|
if active, _ := broker.streams(); active != count {
|
|
t.Fatalf("lost background streams: %d", active)
|
|
}
|
|
}
|
|
|
|
// A readback captured before a newer task event must not erase that task's
|
|
// durable recovery membership, even if its observer has since been released.
|
|
type staleRecoveryTasks struct {
|
|
*recoveryTasks
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
}
|
|
|
|
func (b *staleRecoveryTasks) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
|
|
close(b.entered)
|
|
select {
|
|
case <-b.release:
|
|
return nil, nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
func TestBackgroundRecoveryStaleIdleCannotForgetNewTask(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := openQueueStore(t, restartChatDB(t))
|
|
old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}
|
|
if err := store.RememberChatLiveSession(ctx, old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newChatHub()
|
|
h.liveStore = store
|
|
broker := &staleRecoveryTasks{recoveryTasks: newRecoveryTasks(), entered: make(chan struct{}), release: make(chan struct{})}
|
|
done := make(chan struct{})
|
|
go func() { h.reattachSession(ctx, broker, old); close(done) }()
|
|
<-broker.entered
|
|
h.mu.Lock()
|
|
room := h.roomLocked("ses_1")
|
|
newest := task("new_job")
|
|
h.applyLocked(protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &newest})
|
|
h.releaseRoomLocked(room)
|
|
h.mu.Unlock()
|
|
close(broker.release)
|
|
<-done
|
|
rows, err := store.ChatLiveSessions(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(rows) != 1 || !rows[0].Attention {
|
|
t.Fatalf("stale idle erased new task journal: %+v", rows)
|
|
}
|
|
}
|
|
|
|
func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := openQueueStore(t, restartChatDB(t))
|
|
old := pipeline.ChatLiveSession{SessionID: "ses_1", Attention: true}
|
|
if err := store.RememberChatLiveSession(ctx, old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newChatHub()
|
|
h.liveStore = store
|
|
broker := newRecoveryTasks()
|
|
h.reattachSession(ctx, broker, old)
|
|
rows, err := store.ChatLiveSessions(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(rows) != 0 {
|
|
t.Fatalf("authoritative idle left stale journal: %+v", rows)
|
|
}
|
|
if broker.historyReads != 0 || broker.creates != 0 || broker.promptCount() != 0 {
|
|
t.Fatal("idle recovery started or read a conversation")
|
|
}
|
|
}
|
|
|
|
type blockedRecoveryProbe struct {
|
|
*restartChat
|
|
shellReads atomic.Int32
|
|
once sync.Once
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
}
|
|
|
|
func (b *blockedRecoveryProbe) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
|
|
b.shellReads.Add(1)
|
|
return b.restartChat.Shells(ctx, id)
|
|
}
|
|
func (b *blockedRecoveryProbe) ProbeTurn(ctx context.Context, id string) (bool, error) {
|
|
b.once.Do(func() {
|
|
close(b.entered)
|
|
select {
|
|
case <-b.release:
|
|
case <-ctx.Done():
|
|
}
|
|
})
|
|
return b.restartChat.ProbeTurn(ctx, id)
|
|
}
|
|
func TestChatImmediateSubscribeWaitsForRecoveredTurnOwnership(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
startTurnThenRestart(t, path, true)
|
|
broker := &blockedRecoveryProbe{restartChat: newRestartChat(), entered: make(chan struct{}), release: make(chan struct{})}
|
|
broker.setProbe(true)
|
|
server := startChatRestartServer(t, broker, path)
|
|
<-broker.entered
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
if err := wsjson.Write(server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "early", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
readbackConn := dialChatConn(t, server.ctx, server.srv)
|
|
if err := wsjson.Write(server.ctx, readbackConn, protocol.Message{Type: protocol.ChatShells, ID: "early-shells", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The request is on the live socket while reconciliation is blocked. It
|
|
// must not create a room that makes recovery skip the durable running turn.
|
|
time.Sleep(2 * chatQueueRecoverySettle)
|
|
server.handler.hub.mu.Lock()
|
|
room := server.handler.hub.rooms["ses_1"]
|
|
server.handler.hub.mu.Unlock()
|
|
if room != nil {
|
|
t.Fatal("early subscribe admitted a room before recovery")
|
|
}
|
|
if broker.shellReads.Load() != 0 {
|
|
t.Fatal("early shell readback bypassed recovery admission")
|
|
}
|
|
|
|
close(broker.release)
|
|
for {
|
|
var ack protocol.Message
|
|
if err := readTestFrame(server.ctx, conn, &ack); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ack.Type == protocol.ChatSubscribe && ack.ID == "early" {
|
|
break
|
|
}
|
|
if ack.Type == protocol.Error {
|
|
t.Fatalf("subscribe error: %s", ack.Error)
|
|
}
|
|
}
|
|
time.Sleep(2 * chatQueueRecoverySettle)
|
|
if broker.promptCount() != 0 {
|
|
t.Fatalf("queued prompt delivered into recovered running turn: %v", broker.promptTexts())
|
|
}
|
|
broker.setProbe(false)
|
|
waitFor(t, "exactly one queued prompt delivered after recovered turn", func() bool { return broker.promptCount() == 1 })
|
|
if got := broker.promptTexts(); len(got) != 1 || got[0] != "then ship it" {
|
|
t.Fatalf("wrong or duplicate recovered prompts: %v", got)
|
|
}
|
|
}
|
|
|
|
type failFirstRecoveryStream struct {
|
|
*restartChat
|
|
attempts atomic.Int32
|
|
}
|
|
|
|
func (b *failFirstRecoveryStream) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
|
if b.attempts.Add(1) == 1 {
|
|
return errors.New("temporary initial stream failure")
|
|
}
|
|
return b.restartChat.Stream(ctx, id, ready, emit)
|
|
}
|
|
func TestChatRecoveredForegroundRetriesInitialStreamFailure(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
store := openQueueStore(t, path)
|
|
if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Turn: true, NativeTurnID: "turn_original"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker := &failFirstRecoveryStream{restartChat: newRestartChat()}
|
|
broker.setProbe(true)
|
|
server := startChatRestartServer(t, broker, path)
|
|
waitFor(t, "foreground stream recovered without client", func() bool { active, _ := broker.streams(); return active == 1 && broker.attempts.Load() >= 2 })
|
|
h := server.handler.hub
|
|
h.mu.Lock()
|
|
room := h.rooms["ses_1"]
|
|
owned := room != nil && room.observedTurn && room.nativeTurnID == "turn_original"
|
|
h.mu.Unlock()
|
|
if !owned {
|
|
t.Fatal("stream retry lost recovered foreground ownership")
|
|
}
|
|
if broker.promptCount() != 0 {
|
|
t.Fatal("stream retry prompted model")
|
|
}
|
|
broker.setProbe(false)
|
|
waitFor(t, "recovered foreground authoritative completion", func() bool { h.mu.Lock(); defer h.mu.Unlock(); return h.rooms["ses_1"] == nil })
|
|
}
|