Files
alan a627b38ef3 Merge main; use the shared chatShutdownDetacher for OpenCode and Codex
OpenCode and Codex chats implement Detach (called by chatManager.Close)
instead of a parallel DetachSessions interface. Docs follow the Claude Code
host and the systemd KillMode=process unit. The opt-in live restart check
copies only the stage provider's login row into its scratch OpenCode
database, builds the CLI for skill install and removes its scratch tree.
2026-11-04 16:24:17 +00:00

414 lines
15 KiB
Go

package server
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// TestRestartKeepsRealOpenCodeSessions is the opt-in end-to-end restart check
// against the installed OpenCode binary and a real provider:
//
// PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m
//
// It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch
// database and a free loopback port (with only the operator's opencode-go
// provider login row copied into the scratch OpenCode database), starts a chat turn and a one-stage run that each
// run `sleep 30`, restarts the server mid-turn, and checks both finish
// instead of being aborted. Every scratch process is killed afterwards.
func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
if os.Getenv("PIPELINES_RESTART_LIVE") != "1" {
t.Skip("set PIPELINES_RESTART_LIVE=1 to run the real-OpenCode restart check")
}
realHome, err := os.UserHomeDir()
if err != nil {
t.Fatal(err)
}
root, err := os.MkdirTemp("", "pipelines-restart-live-")
if err != nil {
t.Fatal(err)
}
t.Logf("scratch root %s", root)
// Registered first, so it runs last: after the scratch processes are
// stopped, remove the tree (it holds a copied provider login row).
t.Cleanup(func() { _ = os.RemoveAll(root) })
home := filepath.Join(root, "home")
env := scratchEnvironment(t, realHome, home)
binary := filepath.Join(root, "pipelines-server")
build := exec.Command("go", "build", "-o", binary, "./cmd/pipelines-server")
build.Dir = filepath.Join("..", "..")
if out, err := build.CombinedOutput(); err != nil {
t.Fatalf("build: %v\n%s", err, out)
}
// Chat creation activates the manager skill, so install the current
// source's skills into the scratch harness roots.
cli := filepath.Join(root, "pipelines")
buildCLI := exec.Command("go", "build", "-o", cli, "./cmd/pipelines")
buildCLI.Dir = filepath.Join("..", "..")
if out, err := buildCLI.CombinedOutput(); err != nil {
t.Fatalf("build CLI: %v\n%s", err, out)
}
skills := exec.Command(cli, "skill", "install")
skills.Env = env
if out, err := skills.CombinedOutput(); err != nil {
t.Fatalf("skill install: %v\n%s", err, out)
}
db := filepath.Join(root, "data", "pipelines.db")
if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil {
t.Fatal(err)
}
store, err := pipeline.Open(db)
if err != nil {
t.Fatal(err)
}
if _, err := store.Submit(context.Background(), "restart-live", restartLiveStage); err != nil {
t.Fatal(err)
}
items, _ := store.List(context.Background())
store.Close()
addr := freeLoopback(t)
logPath := filepath.Join(root, "server.log")
server := startScratchServer(t, binary, db, addr, env, logPath)
t.Cleanup(func() {
server.stop()
killScratchServices(t, home)
})
// OpenCode 2.x keeps provider logins in its own database. The chat service
// has just created the scratch one; copy only the stage provider's login
// row into it before the stage service starts (scratch, owner-only).
copyProviderLogin(t, filepath.Join(realHome, ".local", "share", "opencode", "opencode.db"), filepath.Join(home, ".local", "share", "opencode", "opencode.db"), "opencode-go")
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
defer cancel()
chatDir := filepath.Join(root, "chat-work")
runDir := filepath.Join(root, "run-work")
for _, dir := range []string{chatDir, runDir} {
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
}
client := dialLive(t, ctx, addr)
created := client.request(t, ctx, protocol.Message{Type: protocol.ChatCreate, ID: "c1", ChatCreate: &protocol.ChatCreateRequest{Title: "restart live", Directory: chatDir, OperationID: "create-1"}}, protocol.ChatCreate)
if created.ChatSession == nil {
t.Fatalf("chat_create = %+v", created)
}
sessionID := created.ChatSession.ID
prompted := time.Now()
client.request(t, ctx, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: sessionID, OperationID: "op-1", Client: "restart-live",
Text: "Use your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then reply with exactly the text RESTART-CHAT-OK and nothing else."}}, protocol.ChatPrompt)
started := client.request(t, ctx, protocol.Message{Type: protocol.RunPipeline, ID: "r1", PipelineID: items[0].ID, Version: 1, RequestKey: "restart-live", Workspace: runDir, Input: map[string]any{"prompt": "restart live check"}}, protocol.RunStarted)
if started.RunDetail == nil {
t.Fatalf("run_pipeline = %+v", started)
}
runID := started.RunDetail.ID
// Wait until both harness sessions are executing the sleep.
waitLive(t, ctx, "chat shell running", func() bool { return chatShellStatus(t, ctx, addr, sessionID) == "running" })
var stageSession string
waitLive(t, ctx, "stage session admitted", func() bool {
d := getLiveRun(t, ctx, addr, runID)
if len(d.Steps) == 1 && d.Steps[0].SessionID != "" {
stageSession = d.Steps[0].SessionID
return true
}
return false
})
time.Sleep(8 * time.Second) // the stage's shell is inside its sleep
client.close()
t.Log("restarting the scratch server mid-turn")
server.stop()
server = startScratchServer(t, binary, db, addr, env, logPath)
var reply string
waitLiveFor(t, ctx, 5*time.Minute, "chat reply after restart", func() bool {
reply = chatReply(t, ctx, addr, sessionID)
return strings.Contains(reply, "RESTART-CHAT-OK")
})
if status := chatShellStatus(t, ctx, addr, sessionID); status != "completed" {
t.Fatalf("chat shell was not completed after the restart: %q", status)
}
if elapsed := time.Since(prompted); elapsed < 30*time.Second {
t.Fatalf("chat reply arrived before the sleep could finish (%s): the turn was cut short", elapsed)
}
var final *protocol.RunDetail
waitLiveFor(t, ctx, 6*time.Minute, "run finished after restart", func() bool {
final = getLiveRun(t, ctx, addr, runID)
return final.FinishedAt != ""
})
if final.Status != "succeeded" || len(final.Steps) != 1 || final.Steps[0].Status != "completed" || final.Steps[0].SessionID != stageSession {
t.Fatalf("run did not continue on its own session: status=%s diagnostic=%q steps=%+v", final.Status, final.Diagnostic, final.Steps)
}
logData, _ := os.ReadFile(logPath)
for _, want := range []string{"leaving 1 chat session(s) running", "left running for the next start to reattach", "reattached stage one"} {
if !bytes.Contains(logData, []byte(want)) {
t.Errorf("server log lacks %q", want)
}
}
t.Logf("chat reply %q; run %d %s on session %s", reply, runID, final.Status, stageSession)
}
const restartLiveStage = "# stage one\n\n```prompt\nUse your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then write the turn output: a short body saying RESTART-STAGE-OK, with the outcome success.\n```\n\n```tools\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\ntimeout=15m\n```\n"
// scratchEnvironment is the server's environment: a private HOME/XDG tree with
// only the operator's OpenCode provider login and a minimal configuration.
func scratchEnvironment(t *testing.T, realHome, home string) []string {
t.Helper()
config, data, cache := filepath.Join(home, ".config"), filepath.Join(home, ".local", "share"), filepath.Join(home, ".cache")
for _, dir := range []string{filepath.Join(config, "opencode"), filepath.Join(data, "opencode"), cache} {
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
}
// A cold model catalog can leave the first service unable to resolve the
// model; seed the public catalog cache (not a credential).
if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil {
if err := os.MkdirAll(filepath.Join(cache, "opencode"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(cache, "opencode", "models.json"), catalog, 0600); err != nil {
t.Fatal(err)
}
}
settings := `{"$schema":"https://opencode.ai/config.json","model":"opencode-go/deepseek-v4.1-flash","permissions":[{"action":"shell","resource":"*","effect":"allow"}]}`
if err := os.WriteFile(filepath.Join(config, "opencode", "opencode.json"), []byte(settings), 0600); err != nil {
t.Fatal(err)
}
var env []string
for _, entry := range os.Environ() {
key, _, _ := strings.Cut(entry, "=")
switch {
case key == "HOME", strings.HasPrefix(key, "XDG_"), strings.HasPrefix(key, "PIPELINES_"), strings.HasPrefix(key, "OPENCODE_"):
continue
}
env = append(env, entry)
}
return append(env, "HOME="+home, "XDG_CONFIG_HOME="+config, "XDG_DATA_HOME="+data, "XDG_CACHE_HOME="+cache, "XDG_STATE_HOME="+filepath.Join(home, ".local", "state"),
"PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off")
}
func copyProviderLogin(t *testing.T, real, scratch, provider string) {
t.Helper()
if _, err := os.Stat(real); err != nil {
t.Skipf("no OpenCode login database: %v", err)
}
deadline := time.Now().Add(30 * time.Second)
for {
if _, err := os.Stat(scratch); err == nil {
break
}
if time.Now().After(deadline) {
t.Fatal("scratch OpenCode database was not created")
}
time.Sleep(200 * time.Millisecond)
}
script := "ATTACH '" + real + "' AS real; INSERT OR REPLACE INTO credential SELECT * FROM real.credential WHERE integration_id='" + provider + "'; SELECT changes();"
out, err := exec.Command("sqlite3", "-bail", scratch, script).CombinedOutput()
if err != nil || strings.TrimSpace(string(out)) == "0" {
t.Skipf("could not copy the %s login into the scratch OpenCode database: %v %s", provider, err, out)
}
}
func opencodePath() string {
if path, err := exec.LookPath("opencode"); err == nil {
return path
}
return "/opt/homebrew/bin/opencode"
}
type scratchServer struct {
t *testing.T
cmd *exec.Cmd
}
func startScratchServer(t *testing.T, binary, db, addr string, env []string, logPath string) *scratchServer {
t.Helper()
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
t.Fatal(err)
}
cmd := exec.Command(binary, "-db", db, "-addr", addr, "-mdns=false")
cmd.Env, cmd.Stdout, cmd.Stderr = env, logFile, logFile
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
logFile.Close()
s := &scratchServer{t: t, cmd: cmd}
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
if resp, err := http.Get("http://" + addr + "/healthz"); err == nil {
resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return s
}
}
time.Sleep(200 * time.Millisecond)
}
s.stop()
data, _ := os.ReadFile(logPath)
t.Fatalf("scratch server did not become healthy:\n%s", data)
return nil
}
// stop is a service stop/restart: SIGTERM, then wait for a clean exit.
func (s *scratchServer) stop() {
if s == nil || s.cmd == nil || s.cmd.Process == nil || s.cmd.ProcessState != nil {
return
}
_ = s.cmd.Process.Signal(syscall.SIGTERM)
done := make(chan struct{})
go func() { _ = s.cmd.Wait(); close(done) }()
select {
case <-done:
case <-time.After(60 * time.Second):
_ = s.cmd.Process.Kill()
<-done
s.t.Error("scratch server did not stop within 60s of SIGTERM")
}
}
// killScratchServices stops the detached OpenCode services the scratch server
// owned (their pids are in its private registrations).
func killScratchServices(t *testing.T, home string) {
for _, name := range []string{"opencode-stage-service.json", "opencode-chat-service.json"} {
data, err := os.ReadFile(filepath.Join(home, "Library", "Caches", "pipelines", "private", name))
if err != nil {
data, err = os.ReadFile(filepath.Join(home, ".cache", "pipelines", "private", name))
}
if err != nil {
continue
}
var state struct{ PID int }
if json.Unmarshal(data, &state) == nil && state.PID > 0 {
_ = syscall.Kill(state.PID, syscall.SIGTERM)
time.Sleep(time.Second)
_ = syscall.Kill(state.PID, syscall.SIGKILL)
t.Logf("stopped scratch %s (pid %d)", strings.TrimSuffix(name, ".json"), state.PID)
}
}
}
func freeLoopback(t *testing.T) string {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
return "127.0.0.1:" + strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)
}
type liveClient struct{ conn *websocket.Conn }
func dialLive(t *testing.T, ctx context.Context, addr string) *liveClient {
t.Helper()
conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/connect", nil)
if err != nil {
t.Fatal(err)
}
conn.SetReadLimit(16 << 20)
return &liveClient{conn: conn}
}
func (c *liveClient) close() { c.conn.CloseNow() }
func (c *liveClient) request(t *testing.T, ctx context.Context, msg protocol.Message, want string) protocol.Message {
t.Helper()
if err := wsjson.Write(ctx, c.conn, msg); err != nil {
t.Fatal(err)
}
read, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
for {
var reply protocol.Message
if err := wsjson.Read(read, c.conn, &reply); err != nil {
t.Fatalf("waiting for %s: %v", want, err)
}
if reply.ID != msg.ID {
continue
}
if reply.Type == protocol.Error {
t.Fatalf("%s refused: %s", msg.Type, reply.Error)
}
if reply.Type == want {
return reply
}
}
}
func liveQuery(t *testing.T, ctx context.Context, addr string, msg protocol.Message, want string) protocol.Message {
t.Helper()
c := dialLive(t, ctx, addr)
defer c.close()
return c.request(t, ctx, msg, want)
}
func getLiveRun(t *testing.T, ctx context.Context, addr string, id int64) *protocol.RunDetail {
t.Helper()
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.GetRun, ID: "g", RunID: id}, protocol.Run)
if reply.RunDetail == nil {
t.Fatalf("get_run = %+v", reply)
}
return reply.RunDetail
}
func liveMessages(t *testing.T, ctx context.Context, addr, session string) []protocol.ChatMessage {
t.Helper()
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.ChatMessages, ID: "m", ChatQuery: &protocol.ChatQuery{SessionID: session, Limit: 50}}, protocol.ChatMessages)
if reply.ChatHistory != nil {
return reply.ChatHistory.Messages
}
return reply.ChatMessages
}
// chatShellStatus reports the status of the newest shell tool call.
func chatShellStatus(t *testing.T, ctx context.Context, addr, session string) string {
status := ""
for _, message := range liveMessages(t, ctx, addr, session) {
for _, part := range message.Parts {
if part.Type == protocol.ChatPartTool && part.State != nil && strings.Contains(strings.ToLower(part.Name+string(part.State.Input)), "sleep") {
status = part.State.Status
}
}
}
return status
}
func chatReply(t *testing.T, ctx context.Context, addr, session string) string {
_, text := lastReply(liveMessages(t, ctx, addr, session))
return text
}
func waitLive(t *testing.T, ctx context.Context, what string, check func() bool) {
t.Helper()
waitLiveFor(t, ctx, 3*time.Minute, what, check)
}
func waitLiveFor(t *testing.T, ctx context.Context, limit time.Duration, what string, check func() bool) {
t.Helper()
deadline := time.Now().Add(limit)
for time.Now().Before(deadline) && ctx.Err() == nil {
if check() {
return
}
time.Sleep(2 * time.Second)
}
t.Fatal(fmt.Sprintf("timed out waiting for %s", what))
}