package server import ( "bytes" "context" "encoding/json" "fmt" "net" "net/http" "os" "os/exec" "path/filepath" "strconv" "strings" "syscall" "testing" "time" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" ) // TestRestartKeepsRealOpenCodeSessions is the opt-in end-to-end restart check // against the installed OpenCode binary and a real provider: // // PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m // // It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch // database and a free loopback port (with only the operator's opencode-go // provider login row copied into the scratch OpenCode database), starts a chat turn and a one-stage run that each // run `sleep 30`, restarts the server mid-turn, and checks both finish // instead of being aborted. Every scratch process is killed afterwards. func TestRestartKeepsRealOpenCodeSessions(t *testing.T) { if os.Getenv("PIPELINES_RESTART_LIVE") != "1" { t.Skip("set PIPELINES_RESTART_LIVE=1 to run the real-OpenCode restart check") } realHome, err := os.UserHomeDir() if err != nil { t.Fatal(err) } root, err := os.MkdirTemp("", "pipelines-restart-live-") if err != nil { t.Fatal(err) } t.Logf("scratch root %s", root) // Registered first, so it runs last: after the scratch processes are // stopped, remove the tree (it holds a copied provider login row). t.Cleanup(func() { _ = os.RemoveAll(root) }) home := filepath.Join(root, "home") env := scratchEnvironment(t, realHome, home) binary := filepath.Join(root, "pipelines-server") build := exec.Command("go", "build", "-o", binary, "./cmd/pipelines-server") build.Dir = filepath.Join("..", "..") if out, err := build.CombinedOutput(); err != nil { t.Fatalf("build: %v\n%s", err, out) } // Chat creation activates the manager skill, so install the current // source's skills into the scratch harness roots. cli := filepath.Join(root, "pipelines") buildCLI := exec.Command("go", "build", "-o", cli, "./cmd/pipelines") buildCLI.Dir = filepath.Join("..", "..") if out, err := buildCLI.CombinedOutput(); err != nil { t.Fatalf("build CLI: %v\n%s", err, out) } skills := exec.Command(cli, "skill", "install") skills.Env = env if out, err := skills.CombinedOutput(); err != nil { t.Fatalf("skill install: %v\n%s", err, out) } db := filepath.Join(root, "data", "pipelines.db") if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil { t.Fatal(err) } store, err := pipeline.Open(db) if err != nil { t.Fatal(err) } if _, err := store.Submit(context.Background(), "restart-live", restartLiveStage); err != nil { t.Fatal(err) } items, _ := store.List(context.Background()) store.Close() addr := freeLoopback(t) logPath := filepath.Join(root, "server.log") server := startScratchServer(t, binary, db, addr, env, logPath) t.Cleanup(func() { server.stop() killScratchServices(t, home) }) // OpenCode 2.x keeps provider logins in its own database. The chat service // has just created the scratch one; copy only the stage provider's login // row into it before the stage service starts (scratch, owner-only). copyProviderLogin(t, filepath.Join(realHome, ".local", "share", "opencode", "opencode.db"), filepath.Join(home, ".local", "share", "opencode", "opencode.db"), "opencode-go") ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) defer cancel() chatDir := filepath.Join(root, "chat-work") runDir := filepath.Join(root, "run-work") for _, dir := range []string{chatDir, runDir} { if err := os.MkdirAll(dir, 0700); err != nil { t.Fatal(err) } } client := dialLive(t, ctx, addr) created := client.request(t, ctx, protocol.Message{Type: protocol.ChatCreate, ID: "c1", ChatCreate: &protocol.ChatCreateRequest{Title: "restart live", Directory: chatDir, OperationID: "create-1"}}, protocol.ChatCreate) if created.ChatSession == nil { t.Fatalf("chat_create = %+v", created) } sessionID := created.ChatSession.ID prompted := time.Now() client.request(t, ctx, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: sessionID, OperationID: "op-1", Client: "restart-live", Text: "Use your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then reply with exactly the text RESTART-CHAT-OK and nothing else."}}, protocol.ChatPrompt) started := client.request(t, ctx, protocol.Message{Type: protocol.RunPipeline, ID: "r1", PipelineID: items[0].ID, Version: 1, RequestKey: "restart-live", Workspace: runDir, Input: map[string]any{"prompt": "restart live check"}}, protocol.RunStarted) if started.RunDetail == nil { t.Fatalf("run_pipeline = %+v", started) } runID := started.RunDetail.ID // Wait until both harness sessions are executing the sleep. waitLive(t, ctx, "chat shell running", func() bool { return chatShellStatus(t, ctx, addr, sessionID) == "running" }) var stageSession string waitLive(t, ctx, "stage session admitted", func() bool { d := getLiveRun(t, ctx, addr, runID) if len(d.Steps) == 1 && d.Steps[0].SessionID != "" { stageSession = d.Steps[0].SessionID return true } return false }) time.Sleep(8 * time.Second) // the stage's shell is inside its sleep client.close() t.Log("restarting the scratch server mid-turn") server.stop() server = startScratchServer(t, binary, db, addr, env, logPath) var reply string waitLiveFor(t, ctx, 5*time.Minute, "chat reply after restart", func() bool { reply = chatReply(t, ctx, addr, sessionID) return strings.Contains(reply, "RESTART-CHAT-OK") }) if status := chatShellStatus(t, ctx, addr, sessionID); status != "completed" { t.Fatalf("chat shell was not completed after the restart: %q", status) } if elapsed := time.Since(prompted); elapsed < 30*time.Second { t.Fatalf("chat reply arrived before the sleep could finish (%s): the turn was cut short", elapsed) } var final *protocol.RunDetail waitLiveFor(t, ctx, 6*time.Minute, "run finished after restart", func() bool { final = getLiveRun(t, ctx, addr, runID) return final.FinishedAt != "" }) if final.Status != "succeeded" || len(final.Steps) != 1 || final.Steps[0].Status != "completed" || final.Steps[0].SessionID != stageSession { t.Fatalf("run did not continue on its own session: status=%s diagnostic=%q steps=%+v", final.Status, final.Diagnostic, final.Steps) } logData, _ := os.ReadFile(logPath) for _, want := range []string{"leaving 1 chat session(s) running", "left running for the next start to reattach", "reattached stage one"} { if !bytes.Contains(logData, []byte(want)) { t.Errorf("server log lacks %q", want) } } t.Logf("chat reply %q; run %d %s on session %s", reply, runID, final.Status, stageSession) } const restartLiveStage = "# stage one\n\n```prompt\nUse your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then write the turn output: a short body saying RESTART-STAGE-OK, with the outcome success.\n```\n\n```tools\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\ntimeout=15m\n```\n" // scratchEnvironment is the server's environment: a private HOME/XDG tree with // only the operator's OpenCode provider login and a minimal configuration. func scratchEnvironment(t *testing.T, realHome, home string) []string { t.Helper() config, data, cache := filepath.Join(home, ".config"), filepath.Join(home, ".local", "share"), filepath.Join(home, ".cache") for _, dir := range []string{filepath.Join(config, "opencode"), filepath.Join(data, "opencode"), cache} { if err := os.MkdirAll(dir, 0700); err != nil { t.Fatal(err) } } // A cold model catalog can leave the first service unable to resolve the // model; seed the public catalog cache (not a credential). if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil { if err := os.MkdirAll(filepath.Join(cache, "opencode"), 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(cache, "opencode", "models.json"), catalog, 0600); err != nil { t.Fatal(err) } } settings := `{"$schema":"https://opencode.ai/config.json","model":"opencode-go/deepseek-v4.1-flash","permissions":[{"action":"shell","resource":"*","effect":"allow"}]}` if err := os.WriteFile(filepath.Join(config, "opencode", "opencode.json"), []byte(settings), 0600); err != nil { t.Fatal(err) } var env []string for _, entry := range os.Environ() { key, _, _ := strings.Cut(entry, "=") switch { case key == "HOME", strings.HasPrefix(key, "XDG_"), strings.HasPrefix(key, "PIPELINES_"), strings.HasPrefix(key, "OPENCODE_"): continue } env = append(env, entry) } return append(env, "HOME="+home, "XDG_CONFIG_HOME="+config, "XDG_DATA_HOME="+data, "XDG_CACHE_HOME="+cache, "XDG_STATE_HOME="+filepath.Join(home, ".local", "state"), "PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off") } func copyProviderLogin(t *testing.T, real, scratch, provider string) { t.Helper() if _, err := os.Stat(real); err != nil { t.Skipf("no OpenCode login database: %v", err) } deadline := time.Now().Add(30 * time.Second) for { if _, err := os.Stat(scratch); err == nil { break } if time.Now().After(deadline) { t.Fatal("scratch OpenCode database was not created") } time.Sleep(200 * time.Millisecond) } script := "ATTACH '" + real + "' AS real; INSERT OR REPLACE INTO credential SELECT * FROM real.credential WHERE integration_id='" + provider + "'; SELECT changes();" out, err := exec.Command("sqlite3", "-bail", scratch, script).CombinedOutput() if err != nil || strings.TrimSpace(string(out)) == "0" { t.Skipf("could not copy the %s login into the scratch OpenCode database: %v %s", provider, err, out) } } func opencodePath() string { if path, err := exec.LookPath("opencode"); err == nil { return path } return "/opt/homebrew/bin/opencode" } type scratchServer struct { t *testing.T cmd *exec.Cmd } func startScratchServer(t *testing.T, binary, db, addr string, env []string, logPath string) *scratchServer { t.Helper() logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) if err != nil { t.Fatal(err) } cmd := exec.Command(binary, "-db", db, "-addr", addr, "-mdns=false") cmd.Env, cmd.Stdout, cmd.Stderr = env, logFile, logFile if err := cmd.Start(); err != nil { t.Fatal(err) } logFile.Close() s := &scratchServer{t: t, cmd: cmd} deadline := time.Now().Add(90 * time.Second) for time.Now().Before(deadline) { if resp, err := http.Get("http://" + addr + "/healthz"); err == nil { resp.Body.Close() if resp.StatusCode == http.StatusOK { return s } } time.Sleep(200 * time.Millisecond) } s.stop() data, _ := os.ReadFile(logPath) t.Fatalf("scratch server did not become healthy:\n%s", data) return nil } // stop is a service stop/restart: SIGTERM, then wait for a clean exit. func (s *scratchServer) stop() { if s == nil || s.cmd == nil || s.cmd.Process == nil || s.cmd.ProcessState != nil { return } _ = s.cmd.Process.Signal(syscall.SIGTERM) done := make(chan struct{}) go func() { _ = s.cmd.Wait(); close(done) }() select { case <-done: case <-time.After(60 * time.Second): _ = s.cmd.Process.Kill() <-done s.t.Error("scratch server did not stop within 60s of SIGTERM") } } // killScratchServices stops the detached OpenCode services the scratch server // owned (their pids are in its private registrations). func killScratchServices(t *testing.T, home string) { for _, name := range []string{"opencode-stage-service.json", "opencode-chat-service.json"} { data, err := os.ReadFile(filepath.Join(home, "Library", "Caches", "pipelines", "private", name)) if err != nil { data, err = os.ReadFile(filepath.Join(home, ".cache", "pipelines", "private", name)) } if err != nil { continue } var state struct{ PID int } if json.Unmarshal(data, &state) == nil && state.PID > 0 { _ = syscall.Kill(state.PID, syscall.SIGTERM) time.Sleep(time.Second) _ = syscall.Kill(state.PID, syscall.SIGKILL) t.Logf("stopped scratch %s (pid %d)", strings.TrimSuffix(name, ".json"), state.PID) } } } func freeLoopback(t *testing.T) string { t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer listener.Close() return "127.0.0.1:" + strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) } type liveClient struct{ conn *websocket.Conn } func dialLive(t *testing.T, ctx context.Context, addr string) *liveClient { t.Helper() conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/connect", nil) if err != nil { t.Fatal(err) } conn.SetReadLimit(16 << 20) return &liveClient{conn: conn} } func (c *liveClient) close() { c.conn.CloseNow() } func (c *liveClient) request(t *testing.T, ctx context.Context, msg protocol.Message, want string) protocol.Message { t.Helper() if err := wsjson.Write(ctx, c.conn, msg); err != nil { t.Fatal(err) } read, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() for { var reply protocol.Message if err := wsjson.Read(read, c.conn, &reply); err != nil { t.Fatalf("waiting for %s: %v", want, err) } if reply.ID != msg.ID { continue } if reply.Type == protocol.Error { t.Fatalf("%s refused: %s", msg.Type, reply.Error) } if reply.Type == want { return reply } } } func liveQuery(t *testing.T, ctx context.Context, addr string, msg protocol.Message, want string) protocol.Message { t.Helper() c := dialLive(t, ctx, addr) defer c.close() return c.request(t, ctx, msg, want) } func getLiveRun(t *testing.T, ctx context.Context, addr string, id int64) *protocol.RunDetail { t.Helper() reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.GetRun, ID: "g", RunID: id}, protocol.Run) if reply.RunDetail == nil { t.Fatalf("get_run = %+v", reply) } return reply.RunDetail } func liveMessages(t *testing.T, ctx context.Context, addr, session string) []protocol.ChatMessage { t.Helper() reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.ChatMessages, ID: "m", ChatQuery: &protocol.ChatQuery{SessionID: session, Limit: 50}}, protocol.ChatMessages) if reply.ChatHistory != nil { return reply.ChatHistory.Messages } return reply.ChatMessages } // chatShellStatus reports the status of the newest shell tool call. func chatShellStatus(t *testing.T, ctx context.Context, addr, session string) string { status := "" for _, message := range liveMessages(t, ctx, addr, session) { for _, part := range message.Parts { if part.Type == protocol.ChatPartTool && part.State != nil && strings.Contains(strings.ToLower(part.Name+string(part.State.Input)), "sleep") { status = part.State.Status } } } return status } func chatReply(t *testing.T, ctx context.Context, addr, session string) string { _, text := lastReply(liveMessages(t, ctx, addr, session)) return text } func waitLive(t *testing.T, ctx context.Context, what string, check func() bool) { t.Helper() waitLiveFor(t, ctx, 3*time.Minute, what, check) } func waitLiveFor(t *testing.T, ctx context.Context, limit time.Duration, what string, check func() bool) { t.Helper() deadline := time.Now().Add(limit) for time.Now().Before(deadline) && ctx.Err() == nil { if check() { return } time.Sleep(2 * time.Second) } t.Fatal(fmt.Sprintf("timed out waiting for %s", what)) }