Merge startup chat admission recovery gate
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
@@ -161,6 +162,16 @@ func chatClientToken() string {
|
||||
// connection-scoped write function so the read loop and the stream goroutine
|
||||
// never interleave partial websocket writes.
|
||||
func handleChat(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error) bool {
|
||||
// External chat admission and readback must follow durable turn/queue
|
||||
// restoration: even readback can reconcile hub state. Internal recovery
|
||||
// does not dispatch requests here. The cold registry scan runs afterward.
|
||||
if hub != nil && hub.recoveryReady != nil && strings.HasPrefix(string(request.Type), "chat_") {
|
||||
select {
|
||||
case <-hub.recoveryReady:
|
||||
case <-ctx.Done():
|
||||
return true
|
||||
}
|
||||
}
|
||||
switch request.Type {
|
||||
case protocol.ChatPrompt:
|
||||
handleChatPrompt(ctx, request, broker, hub, registry, clientID, write)
|
||||
|
||||
@@ -6,9 +6,12 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket/wsjson"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
@@ -244,3 +247,78 @@ func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T)
|
||||
t.Fatal("idle recovery started or read a conversation")
|
||||
}
|
||||
}
|
||||
|
||||
type blockedRecoveryProbe struct {
|
||||
*restartChat
|
||||
shellReads atomic.Int32
|
||||
once sync.Once
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockedRecoveryProbe) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
|
||||
b.shellReads.Add(1)
|
||||
return b.restartChat.Shells(ctx, id)
|
||||
}
|
||||
func (b *blockedRecoveryProbe) ProbeTurn(ctx context.Context, id string) (bool, error) {
|
||||
b.once.Do(func() {
|
||||
close(b.entered)
|
||||
select {
|
||||
case <-b.release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
})
|
||||
return b.restartChat.ProbeTurn(ctx, id)
|
||||
}
|
||||
func TestChatImmediateSubscribeWaitsForRecoveredTurnOwnership(t *testing.T) {
|
||||
fastReattach(t)
|
||||
path := restartChatDB(t)
|
||||
startTurnThenRestart(t, path, true)
|
||||
broker := &blockedRecoveryProbe{restartChat: newRestartChat(), entered: make(chan struct{}), release: make(chan struct{})}
|
||||
broker.setProbe(true)
|
||||
server := startChatRestartServer(t, broker, path)
|
||||
<-broker.entered
|
||||
conn := dialChatConn(t, server.ctx, server.srv)
|
||||
if err := wsjson.Write(server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "early", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readbackConn := dialChatConn(t, server.ctx, server.srv)
|
||||
if err := wsjson.Write(server.ctx, readbackConn, protocol.Message{Type: protocol.ChatShells, ID: "early-shells", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The request is on the live socket while reconciliation is blocked. It
|
||||
// must not create a room that makes recovery skip the durable running turn.
|
||||
time.Sleep(2 * chatQueueRecoverySettle)
|
||||
server.handler.hub.mu.Lock()
|
||||
room := server.handler.hub.rooms["ses_1"]
|
||||
server.handler.hub.mu.Unlock()
|
||||
if room != nil {
|
||||
t.Fatal("early subscribe admitted a room before recovery")
|
||||
}
|
||||
if broker.shellReads.Load() != 0 {
|
||||
t.Fatal("early shell readback bypassed recovery admission")
|
||||
}
|
||||
|
||||
close(broker.release)
|
||||
for {
|
||||
var ack protocol.Message
|
||||
if err := readTestFrame(server.ctx, conn, &ack); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ack.Type == protocol.ChatSubscribe && ack.ID == "early" {
|
||||
break
|
||||
}
|
||||
if ack.Type == protocol.Error {
|
||||
t.Fatalf("subscribe error: %s", ack.Error)
|
||||
}
|
||||
}
|
||||
time.Sleep(2 * chatQueueRecoverySettle)
|
||||
if broker.promptCount() != 0 {
|
||||
t.Fatalf("queued prompt delivered into recovered running turn: %v", broker.promptTexts())
|
||||
}
|
||||
broker.setProbe(false)
|
||||
waitFor(t, "exactly one queued prompt delivered after recovered turn", func() bool { return broker.promptCount() == 1 })
|
||||
if got := broker.promptTexts(); len(got) != 1 || got[0] != "then ship it" {
|
||||
t.Fatalf("wrong or duplicate recovered prompts: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ type chatHub struct {
|
||||
liveJournal map[string]chatRecoveryState
|
||||
recoveryGeneration map[string]uint64
|
||||
recoveryDone <-chan struct{}
|
||||
recoveryReady <-chan struct{}
|
||||
// recentOps/recentOpOrder are the bounded record of accepted prompt
|
||||
// operation ids, so a client retry is acknowledged instead of re-sent.
|
||||
recentOps map[string]chatRecentOp
|
||||
|
||||
@@ -498,11 +498,14 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
// whose turn is still running, so a reply or question still notifies
|
||||
// with no client attached.
|
||||
done := make(chan struct{})
|
||||
ready := make(chan struct{})
|
||||
hub.recoveryDone = done
|
||||
hub.recoveryReady = ready
|
||||
go func() {
|
||||
hub.reattachSessions(ctx, options.Chat)
|
||||
// Restore live turn ownership before queue rooms can start runners.
|
||||
hub.resumeQueued(ctx, options.Chat)
|
||||
close(ready)
|
||||
hub.observeRunningTurns(ctx, options.Chat)
|
||||
close(done)
|
||||
hub.retryReattachments(ctx, options.Chat)
|
||||
|
||||
Reference in New Issue
Block a user