aa8eddd686
A stopping server no longer interrupts harness sessions. Runs are canceled with a shutdown cause, so stage clients leave admitted sessions running and owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait, read back the result, continue the graph), restarts unadmitted visits, continues runs between stages, and falls back to interrupted with a reason when the harness is gone. Chats with live work are recorded on shutdown and reattached on start; turns that ended during the downtime are folded in from history. The server-spawned V2 chat serve is now a detached owned service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
204 lines
8.1 KiB
Go
204 lines
8.1 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// restartChat is a harness fake whose sessions live outside the server, the
|
|
// way an OpenCode service or the Codex App Server keeps them: a turn keeps
|
|
// running (probe) and its history (Messages) is readable after the server
|
|
// that started it is gone.
|
|
type restartChat struct {
|
|
*observedChat
|
|
hmu sync.Mutex
|
|
history []protocol.ChatMessage
|
|
}
|
|
|
|
func newRestartChat() *restartChat { return &restartChat{observedChat: newObservedChat()} }
|
|
|
|
func (c *restartChat) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) {
|
|
c.hmu.Lock()
|
|
defer c.hmu.Unlock()
|
|
return protocol.ChatHistoryPage{Messages: append([]protocol.ChatMessage(nil), c.history...)}, nil
|
|
}
|
|
|
|
func (c *restartChat) setProbe(running bool) {
|
|
c.smu.Lock()
|
|
c.probe["ses_1"] = running
|
|
c.smu.Unlock()
|
|
}
|
|
|
|
// chatRestartServer is one server process over a shared database.
|
|
type chatRestartServer struct {
|
|
ctx context.Context
|
|
cancel context.CancelFunc
|
|
srv *httptest.Server
|
|
handler *runHandler
|
|
}
|
|
|
|
func startChatRestartServer(t *testing.T, broker chatBroker, path string) *chatRestartServer {
|
|
t.Helper()
|
|
store := openQueueStore(t, path)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
handler := HandlerWithOptions(ctx, Options{Chat: broker, DetachSessions: true}, store).(*runHandler)
|
|
s := &chatRestartServer{ctx: ctx, cancel: cancel, srv: httptest.NewServer(handler), handler: handler}
|
|
t.Cleanup(s.stop)
|
|
return s
|
|
}
|
|
|
|
// shutdown is the server's SIGTERM path: record live chats, then stop.
|
|
func (s *chatRestartServer) shutdown() {
|
|
DetachSessions(s.handler)
|
|
s.stop()
|
|
WaitRuns(s.handler)
|
|
}
|
|
|
|
func (s *chatRestartServer) stop() {
|
|
s.srv.CloseClientConnections()
|
|
s.srv.Close()
|
|
s.cancel()
|
|
}
|
|
|
|
func restartChatDB(t *testing.T) string {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "chat.db")
|
|
seed := openQueueStore(t, path)
|
|
if err := seed.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := seed.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seed.Close()
|
|
return path
|
|
}
|
|
|
|
func fastReattach(t *testing.T) {
|
|
t.Helper()
|
|
fastUnattended(t)
|
|
t.Setenv("PIPELINES_NTFY_URL", "")
|
|
poll := chatReattachPoll
|
|
chatReattachPoll = 20 * time.Millisecond
|
|
t.Cleanup(func() { chatReattachPoll = poll })
|
|
}
|
|
|
|
// startTurnThenRestart prompts on the first server, holds the turn in flight
|
|
// (and optionally queues a second prompt), and shuts that server down.
|
|
func startTurnThenRestart(t *testing.T, path string, queueSecond bool) {
|
|
t.Helper()
|
|
first := newRestartChat()
|
|
server := startChatRestartServer(t, first, path)
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
|
|
first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
|
waitFor(t, "first prompt", func() bool { return first.promptCount() == 1 })
|
|
if queueSecond {
|
|
if ack := sendPrompt(t, server.ctx, conn, "p2", "then ship it", "op-2"); !ack.ChatQueued {
|
|
t.Fatalf("second prompt was not queued behind the running turn: %+v", ack)
|
|
}
|
|
}
|
|
server.shutdown()
|
|
}
|
|
|
|
func replyNotes(t *testing.T, path string) []protocol.AppNotification {
|
|
t.Helper()
|
|
return chatNotes(t, openQueueStore(t, path), protocol.NotificationChatReply)
|
|
}
|
|
|
|
// A turn in flight at shutdown that finished while the server was down is
|
|
// folded in from the harness's history: its reply notifies once, with the
|
|
// text the harness recorded during the downtime.
|
|
func TestChatTurnFinishedDuringRestartIsFoldedIn(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
startTurnThenRestart(t, path, false)
|
|
second := newRestartChat()
|
|
second.setProbe(false) // ended while the server was down
|
|
second.history = []protocol.ChatMessage{
|
|
{ID: "msg_u", SessionID: "ses_1", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "build it"}}},
|
|
{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Built while you were away."}}},
|
|
}
|
|
startChatRestartServer(t, second, path)
|
|
waitFor(t, "reply folded in from history", func() bool { return len(replyNotes(t, path)) == 1 })
|
|
if notes := replyNotes(t, path); !strings.Contains(notes[0].Message, "Built while you were away.") {
|
|
t.Fatalf("reply body not read back from history: %+v", notes[0])
|
|
}
|
|
}
|
|
|
|
// A turn still running after the restart keeps its queued prompt waiting;
|
|
// when the harness reports the turn idle (its end event was lost with the old
|
|
// stream), the end is folded in and the queued prompt is delivered.
|
|
func TestChatQueuedPromptDeliveredAfterRestart(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
startTurnThenRestart(t, path, true)
|
|
second := newRestartChat()
|
|
second.setProbe(true)
|
|
second.history = []protocol.ChatMessage{{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "First part done."}}}}
|
|
startChatRestartServer(t, second, path)
|
|
waitFor(t, "reattached stream", func() bool { active, _ := second.streams(); return active >= 1 })
|
|
time.Sleep(2 * chatQueueRecoverySettle)
|
|
if second.promptCount() != 0 {
|
|
t.Fatalf("queued prompt delivered into a turn still running: %v", second.promptTexts())
|
|
}
|
|
second.setProbe(false)
|
|
waitFor(t, "queued prompt delivered", func() bool { return second.promptCount() == 1 })
|
|
if texts := second.promptTexts(); texts[0] != "then ship it" {
|
|
t.Fatalf("wrong prompt delivered after restart: %v", texts)
|
|
}
|
|
waitFor(t, "reply notified once", func() bool { return len(replyNotes(t, path)) == 1 })
|
|
}
|
|
|
|
// A question pending when the server stopped is observed again after the
|
|
// restart with no client attached, and a client that opens the chat sees it.
|
|
func TestChatPendingQuestionSurvivesRestart(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
question := protocol.ChatPermissionFrame{SessionID: "ses_1", RequestID: "req_1", Action: "bash", Resources: []string{"make release"}}
|
|
first := newRestartChat()
|
|
first.perm = []protocol.ChatPermissionFrame{question}
|
|
server := startChatRestartServer(t, first, path)
|
|
conn := dialChatConn(t, server.ctx, server.srv)
|
|
sendPrompt(t, server.ctx, conn, "p1", "release", "op-1")
|
|
first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
|
|
waitFor(t, "question surfaced", func() bool { return len(server.handler.hub.pendingFrames("ses_1")) == 1 })
|
|
server.shutdown()
|
|
|
|
second := newRestartChat()
|
|
second.setProbe(true)
|
|
second.perm = []protocol.ChatPermissionFrame{question}
|
|
restarted := startChatRestartServer(t, second, path)
|
|
waitFor(t, "question observed after restart", func() bool { return len(restarted.handler.hub.pendingFrames("ses_1")) == 1 })
|
|
// A client opening the chat after the restart is shown the question and
|
|
// its answer reaches the harness.
|
|
phone := dialChatConn(t, restarted.ctx, restarted.srv)
|
|
writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
|
|
readUntil(t, restarted.ctx, phone, protocol.ChatSubscribe)
|
|
if pending := nextPermission(t, restarted.ctx, phone, "req_1"); pending.Resolved {
|
|
t.Fatalf("question resolved by the restart: %+v", pending)
|
|
}
|
|
writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatReply, ID: "a1", ChatReply: &protocol.ChatReplyRequest{SessionID: "ses_1", RequestID: "req_1", Decision: protocol.ChatDecisionOnce, Client: "Phone"}})
|
|
readUntil(t, restarted.ctx, phone, protocol.ChatReply)
|
|
waitFor(t, "answer delivered to the harness", func() bool { return second.replyCount() == 1 })
|
|
}
|
|
|
|
// Nothing was in flight: nothing is recorded and nothing is observed.
|
|
func TestChatIdleShutdownRecordsNothing(t *testing.T) {
|
|
fastReattach(t)
|
|
path := restartChatDB(t)
|
|
server := startChatRestartServer(t, newRestartChat(), path)
|
|
server.shutdown()
|
|
live, err := openQueueStore(t, path).TakeChatLiveSessions(context.Background())
|
|
if err != nil || len(live) != 0 {
|
|
t.Fatalf("idle shutdown recorded live chats: %+v %v", live, err)
|
|
}
|
|
}
|