Retry initial recovered chat stream attachment failures
This commit is contained in:
@@ -322,3 +322,40 @@ func TestChatImmediateSubscribeWaitsForRecoveredTurnOwnership(t *testing.T) {
|
||||
t.Fatalf("wrong or duplicate recovered prompts: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type failFirstRecoveryStream struct {
|
||||
*restartChat
|
||||
attempts atomic.Int32
|
||||
}
|
||||
|
||||
func (b *failFirstRecoveryStream) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
if b.attempts.Add(1) == 1 {
|
||||
return errors.New("temporary initial stream failure")
|
||||
}
|
||||
return b.restartChat.Stream(ctx, id, ready, emit)
|
||||
}
|
||||
func TestChatRecoveredForegroundRetriesInitialStreamFailure(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
store := openQueueStore(t, path)
|
||||
if err := store.RememberChatLiveSession(context.Background(), pipeline.ChatLiveSession{SessionID: "ses_1", Turn: true, NativeTurnID: "turn_original"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
broker := &failFirstRecoveryStream{restartChat: newRestartChat()}
|
||||
broker.setProbe(true)
|
||||
server := startChatRestartServer(t, broker, path)
|
||||
waitFor(t, "foreground stream recovered without client", func() bool { active, _ := broker.streams(); return active == 1 && broker.attempts.Load() >= 2 })
|
||||
h := server.handler.hub
|
||||
h.mu.Lock()
|
||||
room := h.rooms["ses_1"]
|
||||
owned := room != nil && room.observedTurn && room.nativeTurnID == "turn_original"
|
||||
h.mu.Unlock()
|
||||
if !owned {
|
||||
t.Fatal("stream retry lost recovered foreground ownership")
|
||||
}
|
||||
if broker.promptCount() != 0 {
|
||||
t.Fatal("stream retry prompted model")
|
||||
}
|
||||
broker.setProbe(false)
|
||||
waitFor(t, "recovered foreground authoritative completion", func() bool { h.mu.Lock(); defer h.mu.Unlock(); return h.rooms["ses_1"] == nil })
|
||||
}
|
||||
|
||||
@@ -991,15 +991,20 @@ func (h *chatHub) drainStream(room *chatRoom, result <-chan error) {
|
||||
if err != nil && room.ctx.Err() == nil {
|
||||
log.Printf("chat stream session=%s: %v", room.sessionID, err)
|
||||
}
|
||||
// Losing the transport does not end the harness turn. Re-subscribe without
|
||||
// releasing its queue slot; the harness must report authoritative completion.
|
||||
h.retryRoomObservation(room, h.roomBroker(room))
|
||||
}
|
||||
|
||||
// retryRoomObservation also covers initial attachment failures, which have no
|
||||
// drainStream goroutine yet. Keep ownership until authoritative completion.
|
||||
func (h *chatHub) retryRoomObservation(room *chatRoom, broker chatBroker) {
|
||||
for room.ctx.Err() == nil {
|
||||
select {
|
||||
case <-room.ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
if err := h.ensureObserving(room, h.roomBroker(room)); err == nil {
|
||||
if err := h.ensureObserving(room, broker); err == nil {
|
||||
h.startRoomPoller(room, broker)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,6 +276,7 @@ func (h *chatHub) observeUnattended(sessionID string, broker chatBroker) {
|
||||
if err := h.ensureObserving(room, broker); err != nil {
|
||||
if room.ctx.Err() == nil {
|
||||
log.Printf("chat turn recovery observe session=%s failed (%T)", sessionID, err)
|
||||
go h.retryRoomObservation(room, broker)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user