package server import ( "context" "net/http/httptest" "path/filepath" "strings" "sync" "testing" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) // restartChat is a harness fake whose sessions live outside the server, the // way an OpenCode service or the Codex App Server keeps them: a turn keeps // running (probe) and its history (Messages) is readable after the server // that started it is gone. type restartChat struct { *observedChat hmu sync.Mutex history []protocol.ChatMessage } func newRestartChat() *restartChat { return &restartChat{observedChat: newObservedChat()} } func (c *restartChat) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) { c.hmu.Lock() defer c.hmu.Unlock() return protocol.ChatHistoryPage{Messages: append([]protocol.ChatMessage(nil), c.history...)}, nil } func (c *restartChat) setProbe(running bool) { c.smu.Lock() c.probe["ses_1"] = running c.smu.Unlock() } // chatRestartServer is one server process over a shared database. type chatRestartServer struct { ctx context.Context cancel context.CancelFunc srv *httptest.Server handler *runHandler } func startChatRestartServer(t *testing.T, broker chatBroker, path string) *chatRestartServer { t.Helper() store := openQueueStore(t, path) ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) handler := HandlerWithOptions(ctx, Options{Chat: broker, DetachSessions: true}, store).(*runHandler) s := &chatRestartServer{ctx: ctx, cancel: cancel, srv: httptest.NewServer(handler), handler: handler} t.Cleanup(s.stop) return s } // shutdown is the server's SIGTERM path: record live chats, then stop. func (s *chatRestartServer) shutdown() { DetachSessions(s.handler) s.stop() WaitRuns(s.handler) } func (s *chatRestartServer) stop() { s.srv.CloseClientConnections() s.srv.Close() s.cancel() } func restartChatDB(t *testing.T) string { t.Helper() path := filepath.Join(t.TempDir(), "chat.db") seed := openQueueStore(t, path) if err := seed.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil { t.Fatal(err) } if _, err := seed.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil { t.Fatal(err) } seed.Close() return path } func fastReattach(t *testing.T) { t.Helper() fastUnattended(t) t.Setenv("PIPELINES_NTFY_URL", "") poll := chatReattachPoll chatReattachPoll = 20 * time.Millisecond t.Cleanup(func() { chatReattachPoll = poll }) } // startTurnThenRestart prompts on the first server, holds the turn in flight // (and optionally queues a second prompt), and shuts that server down. func startTurnThenRestart(t *testing.T, path string, queueSecond bool) { t.Helper() first := newRestartChat() server := startChatRestartServer(t, first, path) conn := dialChatConn(t, server.ctx, server.srv) sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1") first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) waitFor(t, "first prompt", func() bool { return first.promptCount() == 1 }) if queueSecond { if ack := sendPrompt(t, server.ctx, conn, "p2", "then ship it", "op-2"); !ack.ChatQueued { t.Fatalf("second prompt was not queued behind the running turn: %+v", ack) } } server.shutdown() } func replyNotes(t *testing.T, path string) []protocol.AppNotification { t.Helper() return chatNotes(t, openQueueStore(t, path), protocol.NotificationChatReply) } // A turn in flight at shutdown that finished while the server was down is // folded in from the harness's history: its reply notifies once, with the // text the harness recorded during the downtime. func TestChatTurnFinishedDuringRestartIsFoldedIn(t *testing.T) { fastReattach(t) path := restartChatDB(t) startTurnThenRestart(t, path, false) second := newRestartChat() second.setProbe(false) // ended while the server was down second.history = []protocol.ChatMessage{ {ID: "msg_u", SessionID: "ses_1", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "build it"}}}, {ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Built while you were away."}}}, } startChatRestartServer(t, second, path) waitFor(t, "reply folded in from history", func() bool { return len(replyNotes(t, path)) == 1 }) if notes := replyNotes(t, path); !strings.Contains(notes[0].Message, "Built while you were away.") { t.Fatalf("reply body not read back from history: %+v", notes[0]) } } // A turn still running after the restart keeps its queued prompt waiting; // when the harness reports the turn idle (its end event was lost with the old // stream), the end is folded in and the queued prompt is delivered. func TestChatQueuedPromptDeliveredAfterRestart(t *testing.T) { fastReattach(t) path := restartChatDB(t) startTurnThenRestart(t, path, true) second := newRestartChat() second.setProbe(true) second.history = []protocol.ChatMessage{{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "First part done."}}}} startChatRestartServer(t, second, path) waitFor(t, "reattached stream", func() bool { active, _ := second.streams(); return active >= 1 }) time.Sleep(2 * chatQueueRecoverySettle) if second.promptCount() != 0 { t.Fatalf("queued prompt delivered into a turn still running: %v", second.promptTexts()) } second.setProbe(false) waitFor(t, "queued prompt delivered", func() bool { return second.promptCount() == 1 }) if texts := second.promptTexts(); texts[0] != "then ship it" { t.Fatalf("wrong prompt delivered after restart: %v", texts) } waitFor(t, "reply notified once", func() bool { return len(replyNotes(t, path)) == 1 }) } // A question pending when the server stopped is observed again after the // restart with no client attached, and a client that opens the chat sees it. func TestChatPendingQuestionSurvivesRestart(t *testing.T) { fastReattach(t) path := restartChatDB(t) question := protocol.ChatPermissionFrame{SessionID: "ses_1", RequestID: "req_1", Action: "bash", Resources: []string{"make release"}} first := newRestartChat() first.perm = []protocol.ChatPermissionFrame{question} server := startChatRestartServer(t, first, path) conn := dialChatConn(t, server.ctx, server.srv) sendPrompt(t, server.ctx, conn, "p1", "release", "op-1") first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) waitFor(t, "question surfaced", func() bool { return len(server.handler.hub.pendingFrames("ses_1")) == 1 }) server.shutdown() second := newRestartChat() second.setProbe(true) second.perm = []protocol.ChatPermissionFrame{question} restarted := startChatRestartServer(t, second, path) waitFor(t, "question observed after restart", func() bool { return len(restarted.handler.hub.pendingFrames("ses_1")) == 1 }) // A client opening the chat after the restart is shown the question and // its answer reaches the harness. phone := dialChatConn(t, restarted.ctx, restarted.srv) writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}) readUntil(t, restarted.ctx, phone, protocol.ChatSubscribe) if pending := nextPermission(t, restarted.ctx, phone, "req_1"); pending.Resolved { t.Fatalf("question resolved by the restart: %+v", pending) } writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatReply, ID: "a1", ChatReply: &protocol.ChatReplyRequest{SessionID: "ses_1", RequestID: "req_1", Decision: protocol.ChatDecisionOnce, Client: "Phone"}}) readUntil(t, restarted.ctx, phone, protocol.ChatReply) waitFor(t, "answer delivered to the harness", func() bool { return second.replyCount() == 1 }) } // Nothing was in flight: nothing is recorded and nothing is observed. func TestChatIdleShutdownRecordsNothing(t *testing.T) { fastReattach(t) path := restartChatDB(t) server := startChatRestartServer(t, newRestartChat(), path) server.shutdown() live, err := openQueueStore(t, path).TakeChatLiveSessions(context.Background()) if err != nil || len(live) != 0 { t.Fatalf("idle shutdown recorded live chats: %+v %v", live, err) } }