a627b38ef3
OpenCode and Codex chats implement Detach (called by chatManager.Close) instead of a parallel DetachSessions interface. Docs follow the Claude Code host and the systemd KillMode=process unit. The opt-in live restart check copies only the stage provider's login row into its scratch OpenCode database, builds the CLI for skill install and removes its scratch tree.
414 lines
15 KiB
Go
414 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// TestRestartKeepsRealOpenCodeSessions is the opt-in end-to-end restart check
|
|
// against the installed OpenCode binary and a real provider:
|
|
//
|
|
// PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m
|
|
//
|
|
// It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch
|
|
// database and a free loopback port (with only the operator's opencode-go
|
|
// provider login row copied into the scratch OpenCode database), starts a chat turn and a one-stage run that each
|
|
// run `sleep 30`, restarts the server mid-turn, and checks both finish
|
|
// instead of being aborted. Every scratch process is killed afterwards.
|
|
func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
|
|
if os.Getenv("PIPELINES_RESTART_LIVE") != "1" {
|
|
t.Skip("set PIPELINES_RESTART_LIVE=1 to run the real-OpenCode restart check")
|
|
}
|
|
realHome, err := os.UserHomeDir()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
root, err := os.MkdirTemp("", "pipelines-restart-live-")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf("scratch root %s", root)
|
|
// Registered first, so it runs last: after the scratch processes are
|
|
// stopped, remove the tree (it holds a copied provider login row).
|
|
t.Cleanup(func() { _ = os.RemoveAll(root) })
|
|
home := filepath.Join(root, "home")
|
|
env := scratchEnvironment(t, realHome, home)
|
|
binary := filepath.Join(root, "pipelines-server")
|
|
build := exec.Command("go", "build", "-o", binary, "./cmd/pipelines-server")
|
|
build.Dir = filepath.Join("..", "..")
|
|
if out, err := build.CombinedOutput(); err != nil {
|
|
t.Fatalf("build: %v\n%s", err, out)
|
|
}
|
|
// Chat creation activates the manager skill, so install the current
|
|
// source's skills into the scratch harness roots.
|
|
cli := filepath.Join(root, "pipelines")
|
|
buildCLI := exec.Command("go", "build", "-o", cli, "./cmd/pipelines")
|
|
buildCLI.Dir = filepath.Join("..", "..")
|
|
if out, err := buildCLI.CombinedOutput(); err != nil {
|
|
t.Fatalf("build CLI: %v\n%s", err, out)
|
|
}
|
|
skills := exec.Command(cli, "skill", "install")
|
|
skills.Env = env
|
|
if out, err := skills.CombinedOutput(); err != nil {
|
|
t.Fatalf("skill install: %v\n%s", err, out)
|
|
}
|
|
db := filepath.Join(root, "data", "pipelines.db")
|
|
if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err := pipeline.Open(db)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(context.Background(), "restart-live", restartLiveStage); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(context.Background())
|
|
store.Close()
|
|
addr := freeLoopback(t)
|
|
logPath := filepath.Join(root, "server.log")
|
|
server := startScratchServer(t, binary, db, addr, env, logPath)
|
|
t.Cleanup(func() {
|
|
server.stop()
|
|
killScratchServices(t, home)
|
|
})
|
|
// OpenCode 2.x keeps provider logins in its own database. The chat service
|
|
// has just created the scratch one; copy only the stage provider's login
|
|
// row into it before the stage service starts (scratch, owner-only).
|
|
copyProviderLogin(t, filepath.Join(realHome, ".local", "share", "opencode", "opencode.db"), filepath.Join(home, ".local", "share", "opencode", "opencode.db"), "opencode-go")
|
|
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
|
|
defer cancel()
|
|
|
|
chatDir := filepath.Join(root, "chat-work")
|
|
runDir := filepath.Join(root, "run-work")
|
|
for _, dir := range []string{chatDir, runDir} {
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
client := dialLive(t, ctx, addr)
|
|
created := client.request(t, ctx, protocol.Message{Type: protocol.ChatCreate, ID: "c1", ChatCreate: &protocol.ChatCreateRequest{Title: "restart live", Directory: chatDir, OperationID: "create-1"}}, protocol.ChatCreate)
|
|
if created.ChatSession == nil {
|
|
t.Fatalf("chat_create = %+v", created)
|
|
}
|
|
sessionID := created.ChatSession.ID
|
|
prompted := time.Now()
|
|
client.request(t, ctx, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: sessionID, OperationID: "op-1", Client: "restart-live",
|
|
Text: "Use your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then reply with exactly the text RESTART-CHAT-OK and nothing else."}}, protocol.ChatPrompt)
|
|
started := client.request(t, ctx, protocol.Message{Type: protocol.RunPipeline, ID: "r1", PipelineID: items[0].ID, Version: 1, RequestKey: "restart-live", Workspace: runDir, Input: map[string]any{"prompt": "restart live check"}}, protocol.RunStarted)
|
|
if started.RunDetail == nil {
|
|
t.Fatalf("run_pipeline = %+v", started)
|
|
}
|
|
runID := started.RunDetail.ID
|
|
// Wait until both harness sessions are executing the sleep.
|
|
waitLive(t, ctx, "chat shell running", func() bool { return chatShellStatus(t, ctx, addr, sessionID) == "running" })
|
|
var stageSession string
|
|
waitLive(t, ctx, "stage session admitted", func() bool {
|
|
d := getLiveRun(t, ctx, addr, runID)
|
|
if len(d.Steps) == 1 && d.Steps[0].SessionID != "" {
|
|
stageSession = d.Steps[0].SessionID
|
|
return true
|
|
}
|
|
return false
|
|
})
|
|
time.Sleep(8 * time.Second) // the stage's shell is inside its sleep
|
|
client.close()
|
|
|
|
t.Log("restarting the scratch server mid-turn")
|
|
server.stop()
|
|
server = startScratchServer(t, binary, db, addr, env, logPath)
|
|
|
|
var reply string
|
|
waitLiveFor(t, ctx, 5*time.Minute, "chat reply after restart", func() bool {
|
|
reply = chatReply(t, ctx, addr, sessionID)
|
|
return strings.Contains(reply, "RESTART-CHAT-OK")
|
|
})
|
|
if status := chatShellStatus(t, ctx, addr, sessionID); status != "completed" {
|
|
t.Fatalf("chat shell was not completed after the restart: %q", status)
|
|
}
|
|
if elapsed := time.Since(prompted); elapsed < 30*time.Second {
|
|
t.Fatalf("chat reply arrived before the sleep could finish (%s): the turn was cut short", elapsed)
|
|
}
|
|
var final *protocol.RunDetail
|
|
waitLiveFor(t, ctx, 6*time.Minute, "run finished after restart", func() bool {
|
|
final = getLiveRun(t, ctx, addr, runID)
|
|
return final.FinishedAt != ""
|
|
})
|
|
if final.Status != "succeeded" || len(final.Steps) != 1 || final.Steps[0].Status != "completed" || final.Steps[0].SessionID != stageSession {
|
|
t.Fatalf("run did not continue on its own session: status=%s diagnostic=%q steps=%+v", final.Status, final.Diagnostic, final.Steps)
|
|
}
|
|
logData, _ := os.ReadFile(logPath)
|
|
for _, want := range []string{"leaving 1 chat session(s) running", "left running for the next start to reattach", "reattached stage one"} {
|
|
if !bytes.Contains(logData, []byte(want)) {
|
|
t.Errorf("server log lacks %q", want)
|
|
}
|
|
}
|
|
t.Logf("chat reply %q; run %d %s on session %s", reply, runID, final.Status, stageSession)
|
|
}
|
|
|
|
const restartLiveStage = "# stage one\n\n```prompt\nUse your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then write the turn output: a short body saying RESTART-STAGE-OK, with the outcome success.\n```\n\n```tools\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\ntimeout=15m\n```\n"
|
|
|
|
// scratchEnvironment is the server's environment: a private HOME/XDG tree with
|
|
// only the operator's OpenCode provider login and a minimal configuration.
|
|
func scratchEnvironment(t *testing.T, realHome, home string) []string {
|
|
t.Helper()
|
|
config, data, cache := filepath.Join(home, ".config"), filepath.Join(home, ".local", "share"), filepath.Join(home, ".cache")
|
|
for _, dir := range []string{filepath.Join(config, "opencode"), filepath.Join(data, "opencode"), cache} {
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// A cold model catalog can leave the first service unable to resolve the
|
|
// model; seed the public catalog cache (not a credential).
|
|
if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil {
|
|
if err := os.MkdirAll(filepath.Join(cache, "opencode"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(cache, "opencode", "models.json"), catalog, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
settings := `{"$schema":"https://opencode.ai/config.json","model":"opencode-go/deepseek-v4.1-flash","permissions":[{"action":"shell","resource":"*","effect":"allow"}]}`
|
|
if err := os.WriteFile(filepath.Join(config, "opencode", "opencode.json"), []byte(settings), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var env []string
|
|
for _, entry := range os.Environ() {
|
|
key, _, _ := strings.Cut(entry, "=")
|
|
switch {
|
|
case key == "HOME", strings.HasPrefix(key, "XDG_"), strings.HasPrefix(key, "PIPELINES_"), strings.HasPrefix(key, "OPENCODE_"):
|
|
continue
|
|
}
|
|
env = append(env, entry)
|
|
}
|
|
return append(env, "HOME="+home, "XDG_CONFIG_HOME="+config, "XDG_DATA_HOME="+data, "XDG_CACHE_HOME="+cache, "XDG_STATE_HOME="+filepath.Join(home, ".local", "state"),
|
|
"PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off")
|
|
}
|
|
|
|
func copyProviderLogin(t *testing.T, real, scratch, provider string) {
|
|
t.Helper()
|
|
if _, err := os.Stat(real); err != nil {
|
|
t.Skipf("no OpenCode login database: %v", err)
|
|
}
|
|
deadline := time.Now().Add(30 * time.Second)
|
|
for {
|
|
if _, err := os.Stat(scratch); err == nil {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("scratch OpenCode database was not created")
|
|
}
|
|
time.Sleep(200 * time.Millisecond)
|
|
}
|
|
script := "ATTACH '" + real + "' AS real; INSERT OR REPLACE INTO credential SELECT * FROM real.credential WHERE integration_id='" + provider + "'; SELECT changes();"
|
|
out, err := exec.Command("sqlite3", "-bail", scratch, script).CombinedOutput()
|
|
if err != nil || strings.TrimSpace(string(out)) == "0" {
|
|
t.Skipf("could not copy the %s login into the scratch OpenCode database: %v %s", provider, err, out)
|
|
}
|
|
}
|
|
|
|
func opencodePath() string {
|
|
if path, err := exec.LookPath("opencode"); err == nil {
|
|
return path
|
|
}
|
|
return "/opt/homebrew/bin/opencode"
|
|
}
|
|
|
|
type scratchServer struct {
|
|
t *testing.T
|
|
cmd *exec.Cmd
|
|
}
|
|
|
|
func startScratchServer(t *testing.T, binary, db, addr string, env []string, logPath string) *scratchServer {
|
|
t.Helper()
|
|
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cmd := exec.Command(binary, "-db", db, "-addr", addr, "-mdns=false")
|
|
cmd.Env, cmd.Stdout, cmd.Stderr = env, logFile, logFile
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
logFile.Close()
|
|
s := &scratchServer{t: t, cmd: cmd}
|
|
deadline := time.Now().Add(90 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if resp, err := http.Get("http://" + addr + "/healthz"); err == nil {
|
|
resp.Body.Close()
|
|
if resp.StatusCode == http.StatusOK {
|
|
return s
|
|
}
|
|
}
|
|
time.Sleep(200 * time.Millisecond)
|
|
}
|
|
s.stop()
|
|
data, _ := os.ReadFile(logPath)
|
|
t.Fatalf("scratch server did not become healthy:\n%s", data)
|
|
return nil
|
|
}
|
|
|
|
// stop is a service stop/restart: SIGTERM, then wait for a clean exit.
|
|
func (s *scratchServer) stop() {
|
|
if s == nil || s.cmd == nil || s.cmd.Process == nil || s.cmd.ProcessState != nil {
|
|
return
|
|
}
|
|
_ = s.cmd.Process.Signal(syscall.SIGTERM)
|
|
done := make(chan struct{})
|
|
go func() { _ = s.cmd.Wait(); close(done) }()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(60 * time.Second):
|
|
_ = s.cmd.Process.Kill()
|
|
<-done
|
|
s.t.Error("scratch server did not stop within 60s of SIGTERM")
|
|
}
|
|
}
|
|
|
|
// killScratchServices stops the detached OpenCode services the scratch server
|
|
// owned (their pids are in its private registrations).
|
|
func killScratchServices(t *testing.T, home string) {
|
|
for _, name := range []string{"opencode-stage-service.json", "opencode-chat-service.json"} {
|
|
data, err := os.ReadFile(filepath.Join(home, "Library", "Caches", "pipelines", "private", name))
|
|
if err != nil {
|
|
data, err = os.ReadFile(filepath.Join(home, ".cache", "pipelines", "private", name))
|
|
}
|
|
if err != nil {
|
|
continue
|
|
}
|
|
var state struct{ PID int }
|
|
if json.Unmarshal(data, &state) == nil && state.PID > 0 {
|
|
_ = syscall.Kill(state.PID, syscall.SIGTERM)
|
|
time.Sleep(time.Second)
|
|
_ = syscall.Kill(state.PID, syscall.SIGKILL)
|
|
t.Logf("stopped scratch %s (pid %d)", strings.TrimSuffix(name, ".json"), state.PID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func freeLoopback(t *testing.T) string {
|
|
t.Helper()
|
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer listener.Close()
|
|
return "127.0.0.1:" + strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)
|
|
}
|
|
|
|
type liveClient struct{ conn *websocket.Conn }
|
|
|
|
func dialLive(t *testing.T, ctx context.Context, addr string) *liveClient {
|
|
t.Helper()
|
|
conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/connect", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
conn.SetReadLimit(16 << 20)
|
|
return &liveClient{conn: conn}
|
|
}
|
|
|
|
func (c *liveClient) close() { c.conn.CloseNow() }
|
|
|
|
func (c *liveClient) request(t *testing.T, ctx context.Context, msg protocol.Message, want string) protocol.Message {
|
|
t.Helper()
|
|
if err := wsjson.Write(ctx, c.conn, msg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
read, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancel()
|
|
for {
|
|
var reply protocol.Message
|
|
if err := wsjson.Read(read, c.conn, &reply); err != nil {
|
|
t.Fatalf("waiting for %s: %v", want, err)
|
|
}
|
|
if reply.ID != msg.ID {
|
|
continue
|
|
}
|
|
if reply.Type == protocol.Error {
|
|
t.Fatalf("%s refused: %s", msg.Type, reply.Error)
|
|
}
|
|
if reply.Type == want {
|
|
return reply
|
|
}
|
|
}
|
|
}
|
|
|
|
func liveQuery(t *testing.T, ctx context.Context, addr string, msg protocol.Message, want string) protocol.Message {
|
|
t.Helper()
|
|
c := dialLive(t, ctx, addr)
|
|
defer c.close()
|
|
return c.request(t, ctx, msg, want)
|
|
}
|
|
|
|
func getLiveRun(t *testing.T, ctx context.Context, addr string, id int64) *protocol.RunDetail {
|
|
t.Helper()
|
|
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.GetRun, ID: "g", RunID: id}, protocol.Run)
|
|
if reply.RunDetail == nil {
|
|
t.Fatalf("get_run = %+v", reply)
|
|
}
|
|
return reply.RunDetail
|
|
}
|
|
|
|
func liveMessages(t *testing.T, ctx context.Context, addr, session string) []protocol.ChatMessage {
|
|
t.Helper()
|
|
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.ChatMessages, ID: "m", ChatQuery: &protocol.ChatQuery{SessionID: session, Limit: 50}}, protocol.ChatMessages)
|
|
if reply.ChatHistory != nil {
|
|
return reply.ChatHistory.Messages
|
|
}
|
|
return reply.ChatMessages
|
|
}
|
|
|
|
// chatShellStatus reports the status of the newest shell tool call.
|
|
func chatShellStatus(t *testing.T, ctx context.Context, addr, session string) string {
|
|
status := ""
|
|
for _, message := range liveMessages(t, ctx, addr, session) {
|
|
for _, part := range message.Parts {
|
|
if part.Type == protocol.ChatPartTool && part.State != nil && strings.Contains(strings.ToLower(part.Name+string(part.State.Input)), "sleep") {
|
|
status = part.State.Status
|
|
}
|
|
}
|
|
}
|
|
return status
|
|
}
|
|
|
|
func chatReply(t *testing.T, ctx context.Context, addr, session string) string {
|
|
_, text := lastReply(liveMessages(t, ctx, addr, session))
|
|
return text
|
|
}
|
|
|
|
func waitLive(t *testing.T, ctx context.Context, what string, check func() bool) {
|
|
t.Helper()
|
|
waitLiveFor(t, ctx, 3*time.Minute, what, check)
|
|
}
|
|
|
|
func waitLiveFor(t *testing.T, ctx context.Context, limit time.Duration, what string, check func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(limit)
|
|
for time.Now().Before(deadline) && ctx.Err() == nil {
|
|
if check() {
|
|
return
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
t.Fatal(fmt.Sprintf("timed out waiting for %s", what))
|
|
}
|