aa8eddd686
A stopping server no longer interrupts harness sessions. Runs are canceled with a shutdown cause, so stage clients leave admitted sessions running and owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait, read back the result, continue the graph), restarts unadmitted visits, continues runs between stages, and falls back to interrupted with a reason when the harness is gone. Chats with live work are recorded on shutdown and reattached on start; turns that ended during the downtime are folded in from history. The server-spawned V2 chat serve is now a detached owned service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
386 lines
14 KiB
Go
386 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// TestRestartKeepsRealOpenCodeSessions is the opt-in end-to-end restart check
|
|
// against the installed OpenCode binary and a real provider:
|
|
//
|
|
// PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m
|
|
//
|
|
// It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch
|
|
// database and a free loopback port (with only the operator's OpenCode
|
|
// provider login copied in), starts a chat turn and a one-stage run that each
|
|
// run `sleep 30`, restarts the server mid-turn, and checks both finish
|
|
// instead of being aborted. Every scratch process is killed afterwards.
|
|
func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
|
|
if os.Getenv("PIPELINES_RESTART_LIVE") != "1" {
|
|
t.Skip("set PIPELINES_RESTART_LIVE=1 to run the real-OpenCode restart check")
|
|
}
|
|
realHome, err := os.UserHomeDir()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
root, err := os.MkdirTemp("", "pipelines-restart-live-")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf("scratch root %s", root)
|
|
home := filepath.Join(root, "home")
|
|
env := scratchEnvironment(t, realHome, home)
|
|
binary := filepath.Join(root, "pipelines-server")
|
|
build := exec.Command("go", "build", "-o", binary, "./cmd/pipelines-server")
|
|
build.Dir = filepath.Join("..", "..")
|
|
if out, err := build.CombinedOutput(); err != nil {
|
|
t.Fatalf("build: %v\n%s", err, out)
|
|
}
|
|
// Chat creation activates the manager skill, so install the current
|
|
// source's skills into the scratch harness roots.
|
|
skills := exec.Command("go", "run", "./cmd/pipelines", "skill", "install")
|
|
skills.Dir, skills.Env = filepath.Join("..", ".."), env
|
|
if out, err := skills.CombinedOutput(); err != nil {
|
|
t.Fatalf("skill install: %v\n%s", err, out)
|
|
}
|
|
db := filepath.Join(root, "data", "pipelines.db")
|
|
if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err := pipeline.Open(db)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(context.Background(), "restart-live", restartLiveStage); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(context.Background())
|
|
store.Close()
|
|
addr := freeLoopback(t)
|
|
logPath := filepath.Join(root, "server.log")
|
|
server := startScratchServer(t, binary, db, addr, env, logPath)
|
|
t.Cleanup(func() {
|
|
server.stop()
|
|
killScratchServices(t, home)
|
|
})
|
|
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
|
|
defer cancel()
|
|
|
|
chatDir := filepath.Join(root, "chat-work")
|
|
runDir := filepath.Join(root, "run-work")
|
|
for _, dir := range []string{chatDir, runDir} {
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
client := dialLive(t, ctx, addr)
|
|
created := client.request(t, ctx, protocol.Message{Type: protocol.ChatCreate, ID: "c1", ChatCreate: &protocol.ChatCreateRequest{Title: "restart live", Directory: chatDir, OperationID: "create-1"}}, protocol.ChatCreate)
|
|
if created.ChatSession == nil {
|
|
t.Fatalf("chat_create = %+v", created)
|
|
}
|
|
sessionID := created.ChatSession.ID
|
|
prompted := time.Now()
|
|
client.request(t, ctx, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: sessionID, OperationID: "op-1", Client: "restart-live",
|
|
Text: "Use your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then reply with exactly the text RESTART-CHAT-OK and nothing else."}}, protocol.ChatPrompt)
|
|
started := client.request(t, ctx, protocol.Message{Type: protocol.RunPipeline, ID: "r1", PipelineID: items[0].ID, Version: 1, RequestKey: "restart-live", Workspace: runDir, Input: map[string]any{"prompt": "restart live check"}}, protocol.RunStarted)
|
|
if started.RunDetail == nil {
|
|
t.Fatalf("run_pipeline = %+v", started)
|
|
}
|
|
runID := started.RunDetail.ID
|
|
// Wait until both harness sessions are executing the sleep.
|
|
waitLive(t, ctx, "chat shell running", func() bool { return chatShellStatus(t, ctx, addr, sessionID) == "running" })
|
|
var stageSession string
|
|
waitLive(t, ctx, "stage session admitted", func() bool {
|
|
d := getLiveRun(t, ctx, addr, runID)
|
|
if len(d.Steps) == 1 && d.Steps[0].SessionID != "" {
|
|
stageSession = d.Steps[0].SessionID
|
|
return true
|
|
}
|
|
return false
|
|
})
|
|
time.Sleep(8 * time.Second) // the stage's shell is inside its sleep
|
|
client.close()
|
|
|
|
t.Log("restarting the scratch server mid-turn")
|
|
server.stop()
|
|
server = startScratchServer(t, binary, db, addr, env, logPath)
|
|
|
|
var reply string
|
|
waitLiveFor(t, ctx, 5*time.Minute, "chat reply after restart", func() bool {
|
|
reply = chatReply(t, ctx, addr, sessionID)
|
|
return strings.Contains(reply, "RESTART-CHAT-OK")
|
|
})
|
|
if status := chatShellStatus(t, ctx, addr, sessionID); status != "completed" {
|
|
t.Fatalf("chat shell was not completed after the restart: %q", status)
|
|
}
|
|
if elapsed := time.Since(prompted); elapsed < 30*time.Second {
|
|
t.Fatalf("chat reply arrived before the sleep could finish (%s): the turn was cut short", elapsed)
|
|
}
|
|
var final *protocol.RunDetail
|
|
waitLiveFor(t, ctx, 6*time.Minute, "run finished after restart", func() bool {
|
|
final = getLiveRun(t, ctx, addr, runID)
|
|
return final.FinishedAt != ""
|
|
})
|
|
if final.Status != "succeeded" || len(final.Steps) != 1 || final.Steps[0].Status != "completed" || final.Steps[0].SessionID != stageSession {
|
|
t.Fatalf("run did not continue on its own session: status=%s diagnostic=%q steps=%+v", final.Status, final.Diagnostic, final.Steps)
|
|
}
|
|
logData, _ := os.ReadFile(logPath)
|
|
for _, want := range []string{"leaving 1 chat session(s) running", "left running for the next start to reattach", "reattached stage one"} {
|
|
if !bytes.Contains(logData, []byte(want)) {
|
|
t.Errorf("server log lacks %q", want)
|
|
}
|
|
}
|
|
t.Logf("chat reply %q; run %d %s on session %s", reply, runID, final.Status, stageSession)
|
|
}
|
|
|
|
const restartLiveStage = "# stage one\n\n```prompt\nUse your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then write the turn output: a short body saying RESTART-STAGE-OK, with the outcome success.\n```\n\n```tools\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\ntimeout=15m\n```\n"
|
|
|
|
// scratchEnvironment is the server's environment: a private HOME/XDG tree with
|
|
// only the operator's OpenCode provider login and a minimal configuration.
|
|
func scratchEnvironment(t *testing.T, realHome, home string) []string {
|
|
t.Helper()
|
|
config, data, cache := filepath.Join(home, ".config"), filepath.Join(home, ".local", "share"), filepath.Join(home, ".cache")
|
|
for _, dir := range []string{filepath.Join(config, "opencode"), filepath.Join(data, "opencode"), cache} {
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
auth, err := os.ReadFile(filepath.Join(realHome, ".local", "share", "opencode", "auth.json"))
|
|
if err != nil {
|
|
t.Skipf("no OpenCode login to copy: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(data, "opencode", "auth.json"), auth, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A cold model catalog can leave the first service unable to resolve the
|
|
// model; seed the public catalog cache (not a credential).
|
|
if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil {
|
|
if err := os.MkdirAll(filepath.Join(cache, "opencode"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(cache, "opencode", "models.json"), catalog, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
settings := `{"$schema":"https://opencode.ai/config.json","model":"opencode-go/deepseek-v4.1-flash","permissions":[{"action":"shell","resource":"*","effect":"allow"}]}`
|
|
if err := os.WriteFile(filepath.Join(config, "opencode", "opencode.json"), []byte(settings), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var env []string
|
|
for _, entry := range os.Environ() {
|
|
key, _, _ := strings.Cut(entry, "=")
|
|
switch {
|
|
case key == "HOME", strings.HasPrefix(key, "XDG_"), strings.HasPrefix(key, "PIPELINES_"), strings.HasPrefix(key, "OPENCODE_"):
|
|
continue
|
|
}
|
|
env = append(env, entry)
|
|
}
|
|
return append(env, "HOME="+home, "XDG_CONFIG_HOME="+config, "XDG_DATA_HOME="+data, "XDG_CACHE_HOME="+cache, "XDG_STATE_HOME="+filepath.Join(home, ".local", "state"),
|
|
"PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off")
|
|
}
|
|
|
|
func opencodePath() string {
|
|
if path, err := exec.LookPath("opencode"); err == nil {
|
|
return path
|
|
}
|
|
return "/opt/homebrew/bin/opencode"
|
|
}
|
|
|
|
type scratchServer struct {
|
|
t *testing.T
|
|
cmd *exec.Cmd
|
|
}
|
|
|
|
func startScratchServer(t *testing.T, binary, db, addr string, env []string, logPath string) *scratchServer {
|
|
t.Helper()
|
|
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cmd := exec.Command(binary, "-db", db, "-addr", addr, "-mdns=false")
|
|
cmd.Env, cmd.Stdout, cmd.Stderr = env, logFile, logFile
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
logFile.Close()
|
|
s := &scratchServer{t: t, cmd: cmd}
|
|
deadline := time.Now().Add(90 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if resp, err := http.Get("http://" + addr + "/healthz"); err == nil {
|
|
resp.Body.Close()
|
|
if resp.StatusCode == http.StatusOK {
|
|
return s
|
|
}
|
|
}
|
|
time.Sleep(200 * time.Millisecond)
|
|
}
|
|
s.stop()
|
|
data, _ := os.ReadFile(logPath)
|
|
t.Fatalf("scratch server did not become healthy:\n%s", data)
|
|
return nil
|
|
}
|
|
|
|
// stop is a service stop/restart: SIGTERM, then wait for a clean exit.
|
|
func (s *scratchServer) stop() {
|
|
if s == nil || s.cmd == nil || s.cmd.Process == nil || s.cmd.ProcessState != nil {
|
|
return
|
|
}
|
|
_ = s.cmd.Process.Signal(syscall.SIGTERM)
|
|
done := make(chan struct{})
|
|
go func() { _ = s.cmd.Wait(); close(done) }()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(60 * time.Second):
|
|
_ = s.cmd.Process.Kill()
|
|
<-done
|
|
s.t.Error("scratch server did not stop within 60s of SIGTERM")
|
|
}
|
|
}
|
|
|
|
// killScratchServices stops the detached OpenCode services the scratch server
|
|
// owned (their pids are in its private registrations).
|
|
func killScratchServices(t *testing.T, home string) {
|
|
for _, name := range []string{"opencode-stage-service.json", "opencode-chat-service.json"} {
|
|
data, err := os.ReadFile(filepath.Join(home, "Library", "Caches", "pipelines", "private", name))
|
|
if err != nil {
|
|
data, err = os.ReadFile(filepath.Join(home, ".cache", "pipelines", "private", name))
|
|
}
|
|
if err != nil {
|
|
continue
|
|
}
|
|
var state struct{ PID int }
|
|
if json.Unmarshal(data, &state) == nil && state.PID > 0 {
|
|
_ = syscall.Kill(state.PID, syscall.SIGTERM)
|
|
time.Sleep(time.Second)
|
|
_ = syscall.Kill(state.PID, syscall.SIGKILL)
|
|
t.Logf("stopped scratch %s (pid %d)", strings.TrimSuffix(name, ".json"), state.PID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func freeLoopback(t *testing.T) string {
|
|
t.Helper()
|
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer listener.Close()
|
|
return "127.0.0.1:" + strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)
|
|
}
|
|
|
|
type liveClient struct{ conn *websocket.Conn }
|
|
|
|
func dialLive(t *testing.T, ctx context.Context, addr string) *liveClient {
|
|
t.Helper()
|
|
conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/connect", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
conn.SetReadLimit(16 << 20)
|
|
return &liveClient{conn: conn}
|
|
}
|
|
|
|
func (c *liveClient) close() { c.conn.CloseNow() }
|
|
|
|
func (c *liveClient) request(t *testing.T, ctx context.Context, msg protocol.Message, want string) protocol.Message {
|
|
t.Helper()
|
|
if err := wsjson.Write(ctx, c.conn, msg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
read, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancel()
|
|
for {
|
|
var reply protocol.Message
|
|
if err := wsjson.Read(read, c.conn, &reply); err != nil {
|
|
t.Fatalf("waiting for %s: %v", want, err)
|
|
}
|
|
if reply.ID != msg.ID {
|
|
continue
|
|
}
|
|
if reply.Type == protocol.Error {
|
|
t.Fatalf("%s refused: %s", msg.Type, reply.Error)
|
|
}
|
|
if reply.Type == want {
|
|
return reply
|
|
}
|
|
}
|
|
}
|
|
|
|
func liveQuery(t *testing.T, ctx context.Context, addr string, msg protocol.Message, want string) protocol.Message {
|
|
t.Helper()
|
|
c := dialLive(t, ctx, addr)
|
|
defer c.close()
|
|
return c.request(t, ctx, msg, want)
|
|
}
|
|
|
|
func getLiveRun(t *testing.T, ctx context.Context, addr string, id int64) *protocol.RunDetail {
|
|
t.Helper()
|
|
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.GetRun, ID: "g", RunID: id}, protocol.Run)
|
|
if reply.RunDetail == nil {
|
|
t.Fatalf("get_run = %+v", reply)
|
|
}
|
|
return reply.RunDetail
|
|
}
|
|
|
|
func liveMessages(t *testing.T, ctx context.Context, addr, session string) []protocol.ChatMessage {
|
|
t.Helper()
|
|
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.ChatMessages, ID: "m", ChatQuery: &protocol.ChatQuery{SessionID: session, Limit: 50}}, protocol.ChatMessages)
|
|
if reply.ChatHistory != nil {
|
|
return reply.ChatHistory.Messages
|
|
}
|
|
return reply.ChatMessages
|
|
}
|
|
|
|
// chatShellStatus reports the status of the newest shell tool call.
|
|
func chatShellStatus(t *testing.T, ctx context.Context, addr, session string) string {
|
|
status := ""
|
|
for _, message := range liveMessages(t, ctx, addr, session) {
|
|
for _, part := range message.Parts {
|
|
if part.Type == protocol.ChatPartTool && part.State != nil && strings.Contains(strings.ToLower(part.Name+string(part.State.Input)), "sleep") {
|
|
status = part.State.Status
|
|
}
|
|
}
|
|
}
|
|
return status
|
|
}
|
|
|
|
func chatReply(t *testing.T, ctx context.Context, addr, session string) string {
|
|
_, text := lastReply(liveMessages(t, ctx, addr, session))
|
|
return text
|
|
}
|
|
|
|
func waitLive(t *testing.T, ctx context.Context, what string, check func() bool) {
|
|
t.Helper()
|
|
waitLiveFor(t, ctx, 3*time.Minute, what, check)
|
|
}
|
|
|
|
func waitLiveFor(t *testing.T, ctx context.Context, limit time.Duration, what string, check func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(limit)
|
|
for time.Now().Before(deadline) && ctx.Err() == nil {
|
|
if check() {
|
|
return
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
t.Fatal(fmt.Sprintf("timed out waiting for %s", what))
|
|
}
|