Files
alan aa8eddd686 Keep OpenCode/Codex chat and stage sessions running across server restarts
A stopping server no longer interrupts harness sessions. Runs are canceled
with a shutdown cause, so stage clients leave admitted sessions running and
owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait,
read back the result, continue the graph), restarts unadmitted visits,
continues runs between stages, and falls back to interrupted with a reason
when the harness is gone. Chats with live work are recorded on shutdown and
reattached on start; turns that ended during the downtime are folded in
from history. The server-spawned V2 chat serve is now a detached owned
service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
2026-11-04 16:17:08 +00:00

204 lines
8.1 KiB
Go

package server
import (
"context"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// restartChat is a harness fake whose sessions live outside the server, the
// way an OpenCode service or the Codex App Server keeps them: a turn keeps
// running (probe) and its history (Messages) is readable after the server
// that started it is gone.
type restartChat struct {
*observedChat
hmu sync.Mutex
history []protocol.ChatMessage
}
func newRestartChat() *restartChat { return &restartChat{observedChat: newObservedChat()} }
func (c *restartChat) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) {
c.hmu.Lock()
defer c.hmu.Unlock()
return protocol.ChatHistoryPage{Messages: append([]protocol.ChatMessage(nil), c.history...)}, nil
}
func (c *restartChat) setProbe(running bool) {
c.smu.Lock()
c.probe["ses_1"] = running
c.smu.Unlock()
}
// chatRestartServer is one server process over a shared database.
type chatRestartServer struct {
ctx context.Context
cancel context.CancelFunc
srv *httptest.Server
handler *runHandler
}
func startChatRestartServer(t *testing.T, broker chatBroker, path string) *chatRestartServer {
t.Helper()
store := openQueueStore(t, path)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
handler := HandlerWithOptions(ctx, Options{Chat: broker, DetachSessions: true}, store).(*runHandler)
s := &chatRestartServer{ctx: ctx, cancel: cancel, srv: httptest.NewServer(handler), handler: handler}
t.Cleanup(s.stop)
return s
}
// shutdown is the server's SIGTERM path: record live chats, then stop.
func (s *chatRestartServer) shutdown() {
DetachSessions(s.handler)
s.stop()
WaitRuns(s.handler)
}
func (s *chatRestartServer) stop() {
s.srv.CloseClientConnections()
s.srv.Close()
s.cancel()
}
func restartChatDB(t *testing.T) string {
t.Helper()
path := filepath.Join(t.TempDir(), "chat.db")
seed := openQueueStore(t, path)
if err := seed.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
if _, err := seed.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
t.Fatal(err)
}
seed.Close()
return path
}
func fastReattach(t *testing.T) {
t.Helper()
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
poll := chatReattachPoll
chatReattachPoll = 20 * time.Millisecond
t.Cleanup(func() { chatReattachPoll = poll })
}
// startTurnThenRestart prompts on the first server, holds the turn in flight
// (and optionally queues a second prompt), and shuts that server down.
func startTurnThenRestart(t *testing.T, path string, queueSecond bool) {
t.Helper()
first := newRestartChat()
server := startChatRestartServer(t, first, path)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
waitFor(t, "first prompt", func() bool { return first.promptCount() == 1 })
if queueSecond {
if ack := sendPrompt(t, server.ctx, conn, "p2", "then ship it", "op-2"); !ack.ChatQueued {
t.Fatalf("second prompt was not queued behind the running turn: %+v", ack)
}
}
server.shutdown()
}
func replyNotes(t *testing.T, path string) []protocol.AppNotification {
t.Helper()
return chatNotes(t, openQueueStore(t, path), protocol.NotificationChatReply)
}
// A turn in flight at shutdown that finished while the server was down is
// folded in from the harness's history: its reply notifies once, with the
// text the harness recorded during the downtime.
func TestChatTurnFinishedDuringRestartIsFoldedIn(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
startTurnThenRestart(t, path, false)
second := newRestartChat()
second.setProbe(false) // ended while the server was down
second.history = []protocol.ChatMessage{
{ID: "msg_u", SessionID: "ses_1", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "build it"}}},
{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Built while you were away."}}},
}
startChatRestartServer(t, second, path)
waitFor(t, "reply folded in from history", func() bool { return len(replyNotes(t, path)) == 1 })
if notes := replyNotes(t, path); !strings.Contains(notes[0].Message, "Built while you were away.") {
t.Fatalf("reply body not read back from history: %+v", notes[0])
}
}
// A turn still running after the restart keeps its queued prompt waiting;
// when the harness reports the turn idle (its end event was lost with the old
// stream), the end is folded in and the queued prompt is delivered.
func TestChatQueuedPromptDeliveredAfterRestart(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
startTurnThenRestart(t, path, true)
second := newRestartChat()
second.setProbe(true)
second.history = []protocol.ChatMessage{{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "First part done."}}}}
startChatRestartServer(t, second, path)
waitFor(t, "reattached stream", func() bool { active, _ := second.streams(); return active >= 1 })
time.Sleep(2 * chatQueueRecoverySettle)
if second.promptCount() != 0 {
t.Fatalf("queued prompt delivered into a turn still running: %v", second.promptTexts())
}
second.setProbe(false)
waitFor(t, "queued prompt delivered", func() bool { return second.promptCount() == 1 })
if texts := second.promptTexts(); texts[0] != "then ship it" {
t.Fatalf("wrong prompt delivered after restart: %v", texts)
}
waitFor(t, "reply notified once", func() bool { return len(replyNotes(t, path)) == 1 })
}
// A question pending when the server stopped is observed again after the
// restart with no client attached, and a client that opens the chat sees it.
func TestChatPendingQuestionSurvivesRestart(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
question := protocol.ChatPermissionFrame{SessionID: "ses_1", RequestID: "req_1", Action: "bash", Resources: []string{"make release"}}
first := newRestartChat()
first.perm = []protocol.ChatPermissionFrame{question}
server := startChatRestartServer(t, first, path)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "release", "op-1")
first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
waitFor(t, "question surfaced", func() bool { return len(server.handler.hub.pendingFrames("ses_1")) == 1 })
server.shutdown()
second := newRestartChat()
second.setProbe(true)
second.perm = []protocol.ChatPermissionFrame{question}
restarted := startChatRestartServer(t, second, path)
waitFor(t, "question observed after restart", func() bool { return len(restarted.handler.hub.pendingFrames("ses_1")) == 1 })
// A client opening the chat after the restart is shown the question and
// its answer reaches the harness.
phone := dialChatConn(t, restarted.ctx, restarted.srv)
writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, restarted.ctx, phone, protocol.ChatSubscribe)
if pending := nextPermission(t, restarted.ctx, phone, "req_1"); pending.Resolved {
t.Fatalf("question resolved by the restart: %+v", pending)
}
writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatReply, ID: "a1", ChatReply: &protocol.ChatReplyRequest{SessionID: "ses_1", RequestID: "req_1", Decision: protocol.ChatDecisionOnce, Client: "Phone"}})
readUntil(t, restarted.ctx, phone, protocol.ChatReply)
waitFor(t, "answer delivered to the harness", func() bool { return second.replyCount() == 1 })
}
// Nothing was in flight: nothing is recorded and nothing is observed.
func TestChatIdleShutdownRecordsNothing(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
server := startChatRestartServer(t, newRestartChat(), path)
server.shutdown()
live, err := openQueueStore(t, path).TakeChatLiveSessions(context.Background())
if err != nil || len(live) != 0 {
t.Fatalf("idle shutdown recorded live chats: %+v %v", live, err)
}
}