Merge startup chat admission recovery gate

This commit is contained in:
2026-11-07 16:41:58 +00:00
4 changed files with 93 additions and 0 deletions
+11
View File
@@ -7,6 +7,7 @@ import (
"git.a-wan.net/alan/pipelines/internal/opencode"
"regexp"
"strconv"
"strings"
"sync/atomic"
"git.a-wan.net/alan/pipelines/internal/pipeline"
@@ -161,6 +162,16 @@ func chatClientToken() string {
// connection-scoped write function so the read loop and the stream goroutine
// never interleave partial websocket writes.
func handleChat(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error) bool {
// External chat admission and readback must follow durable turn/queue
// restoration: even readback can reconcile hub state. Internal recovery
// does not dispatch requests here. The cold registry scan runs afterward.
if hub != nil && hub.recoveryReady != nil && strings.HasPrefix(string(request.Type), "chat_") {
select {
case <-hub.recoveryReady:
case <-ctx.Done():
return true
}
}
switch request.Type {
case protocol.ChatPrompt:
handleChatPrompt(ctx, request, broker, hub, registry, clientID, write)
@@ -6,9 +6,12 @@ import (
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/coder/websocket/wsjson"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
@@ -244,3 +247,78 @@ func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T)
t.Fatal("idle recovery started or read a conversation")
}
}
type blockedRecoveryProbe struct {
*restartChat
shellReads atomic.Int32
once sync.Once
entered chan struct{}
release chan struct{}
}
func (b *blockedRecoveryProbe) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
b.shellReads.Add(1)
return b.restartChat.Shells(ctx, id)
}
func (b *blockedRecoveryProbe) ProbeTurn(ctx context.Context, id string) (bool, error) {
b.once.Do(func() {
close(b.entered)
select {
case <-b.release:
case <-ctx.Done():
}
})
return b.restartChat.ProbeTurn(ctx, id)
}
func TestChatImmediateSubscribeWaitsForRecoveredTurnOwnership(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
startTurnThenRestart(t, path, true)
broker := &blockedRecoveryProbe{restartChat: newRestartChat(), entered: make(chan struct{}), release: make(chan struct{})}
broker.setProbe(true)
server := startChatRestartServer(t, broker, path)
<-broker.entered
conn := dialChatConn(t, server.ctx, server.srv)
if err := wsjson.Write(server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "early", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
t.Fatal(err)
}
readbackConn := dialChatConn(t, server.ctx, server.srv)
if err := wsjson.Write(server.ctx, readbackConn, protocol.Message{Type: protocol.ChatShells, ID: "early-shells", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
t.Fatal(err)
}
// The request is on the live socket while reconciliation is blocked. It
// must not create a room that makes recovery skip the durable running turn.
time.Sleep(2 * chatQueueRecoverySettle)
server.handler.hub.mu.Lock()
room := server.handler.hub.rooms["ses_1"]
server.handler.hub.mu.Unlock()
if room != nil {
t.Fatal("early subscribe admitted a room before recovery")
}
if broker.shellReads.Load() != 0 {
t.Fatal("early shell readback bypassed recovery admission")
}
close(broker.release)
for {
var ack protocol.Message
if err := readTestFrame(server.ctx, conn, &ack); err != nil {
t.Fatal(err)
}
if ack.Type == protocol.ChatSubscribe && ack.ID == "early" {
break
}
if ack.Type == protocol.Error {
t.Fatalf("subscribe error: %s", ack.Error)
}
}
time.Sleep(2 * chatQueueRecoverySettle)
if broker.promptCount() != 0 {
t.Fatalf("queued prompt delivered into recovered running turn: %v", broker.promptTexts())
}
broker.setProbe(false)
waitFor(t, "exactly one queued prompt delivered after recovered turn", func() bool { return broker.promptCount() == 1 })
if got := broker.promptTexts(); len(got) != 1 || got[0] != "then ship it" {
t.Fatalf("wrong or duplicate recovered prompts: %v", got)
}
}
+1
View File
@@ -60,6 +60,7 @@ type chatHub struct {
liveJournal map[string]chatRecoveryState
recoveryGeneration map[string]uint64
recoveryDone <-chan struct{}
recoveryReady <-chan struct{}
// recentOps/recentOpOrder are the bounded record of accepted prompt
// operation ids, so a client retry is acknowledged instead of re-sent.
recentOps map[string]chatRecentOp
+3
View File
@@ -498,11 +498,14 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
// whose turn is still running, so a reply or question still notifies
// with no client attached.
done := make(chan struct{})
ready := make(chan struct{})
hub.recoveryDone = done
hub.recoveryReady = ready
go func() {
hub.reattachSessions(ctx, options.Chat)
// Restore live turn ownership before queue rooms can start runners.
hub.resumeQueued(ctx, options.Chat)
close(ready)
hub.observeRunningTurns(ctx, options.Chat)
close(done)
hub.retryReattachments(ctx, options.Chat)