diff --git a/internal/server/chat.go b/internal/server/chat.go index d859cffd..d6948f3e 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -7,6 +7,7 @@ import ( "git.a-wan.net/alan/pipelines/internal/opencode" "regexp" "strconv" + "strings" "sync/atomic" "git.a-wan.net/alan/pipelines/internal/pipeline" @@ -161,6 +162,16 @@ func chatClientToken() string { // connection-scoped write function so the read loop and the stream goroutine // never interleave partial websocket writes. func handleChat(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error) bool { + // External chat admission and readback must follow durable turn/queue + // restoration: even readback can reconcile hub state. Internal recovery + // does not dispatch requests here. The cold registry scan runs afterward. + if hub != nil && hub.recoveryReady != nil && strings.HasPrefix(string(request.Type), "chat_") { + select { + case <-hub.recoveryReady: + case <-ctx.Done(): + return true + } + } switch request.Type { case protocol.ChatPrompt: handleChatPrompt(ctx, request, broker, hub, registry, clientID, write) diff --git a/internal/server/chat_background_recovery_test.go b/internal/server/chat_background_recovery_test.go index 972214e2..19b3c942 100644 --- a/internal/server/chat_background_recovery_test.go +++ b/internal/server/chat_background_recovery_test.go @@ -6,9 +6,12 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "testing" "time" + "github.com/coder/websocket/wsjson" + "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" ) @@ -244,3 +247,78 @@ func TestBackgroundRecoveryAuthoritativeIdleClearsStaleMembership(t *testing.T) t.Fatal("idle recovery started or read a conversation") } } + +type blockedRecoveryProbe struct { + *restartChat + shellReads atomic.Int32 + once sync.Once + entered chan struct{} + release chan struct{} +} + +func (b *blockedRecoveryProbe) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) { + b.shellReads.Add(1) + return b.restartChat.Shells(ctx, id) +} +func (b *blockedRecoveryProbe) ProbeTurn(ctx context.Context, id string) (bool, error) { + b.once.Do(func() { + close(b.entered) + select { + case <-b.release: + case <-ctx.Done(): + } + }) + return b.restartChat.ProbeTurn(ctx, id) +} +func TestChatImmediateSubscribeWaitsForRecoveredTurnOwnership(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + startTurnThenRestart(t, path, true) + broker := &blockedRecoveryProbe{restartChat: newRestartChat(), entered: make(chan struct{}), release: make(chan struct{})} + broker.setProbe(true) + server := startChatRestartServer(t, broker, path) + <-broker.entered + conn := dialChatConn(t, server.ctx, server.srv) + if err := wsjson.Write(server.ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "early", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + readbackConn := dialChatConn(t, server.ctx, server.srv) + if err := wsjson.Write(server.ctx, readbackConn, protocol.Message{Type: protocol.ChatShells, ID: "early-shells", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + // The request is on the live socket while reconciliation is blocked. It + // must not create a room that makes recovery skip the durable running turn. + time.Sleep(2 * chatQueueRecoverySettle) + server.handler.hub.mu.Lock() + room := server.handler.hub.rooms["ses_1"] + server.handler.hub.mu.Unlock() + if room != nil { + t.Fatal("early subscribe admitted a room before recovery") + } + if broker.shellReads.Load() != 0 { + t.Fatal("early shell readback bypassed recovery admission") + } + + close(broker.release) + for { + var ack protocol.Message + if err := readTestFrame(server.ctx, conn, &ack); err != nil { + t.Fatal(err) + } + if ack.Type == protocol.ChatSubscribe && ack.ID == "early" { + break + } + if ack.Type == protocol.Error { + t.Fatalf("subscribe error: %s", ack.Error) + } + } + time.Sleep(2 * chatQueueRecoverySettle) + if broker.promptCount() != 0 { + t.Fatalf("queued prompt delivered into recovered running turn: %v", broker.promptTexts()) + } + broker.setProbe(false) + waitFor(t, "exactly one queued prompt delivered after recovered turn", func() bool { return broker.promptCount() == 1 }) + if got := broker.promptTexts(); len(got) != 1 || got[0] != "then ship it" { + t.Fatalf("wrong or duplicate recovered prompts: %v", got) + } +} diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 18b2a8d9..71686418 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -60,6 +60,7 @@ type chatHub struct { liveJournal map[string]chatRecoveryState recoveryGeneration map[string]uint64 recoveryDone <-chan struct{} + recoveryReady <-chan struct{} // recentOps/recentOpOrder are the bounded record of accepted prompt // operation ids, so a client retry is acknowledged instead of re-sent. recentOps map[string]chatRecentOp diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 633429e8..1b6f752c 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -498,11 +498,14 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin // whose turn is still running, so a reply or question still notifies // with no client attached. done := make(chan struct{}) + ready := make(chan struct{}) hub.recoveryDone = done + hub.recoveryReady = ready go func() { hub.reattachSessions(ctx, options.Chat) // Restore live turn ownership before queue rooms can start runners. hub.resumeQueued(ctx, options.Chat) + close(ready) hub.observeRunningTurns(ctx, options.Chat) close(done) hub.retryReattachments(ctx, options.Chat)