diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index 27ef24a2..87c5e05e 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -49,8 +49,18 @@ func main() { if err := store.AcquireServerLock(); err != nil { log.Fatal(err) } - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + signals, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() + // The server's own context ends after a signal, once the chats left live on + // their harness have been recorded (see the shutdown goroutine below). + ctx, cancelServer := context.WithCancel(context.Background()) + defer cancelServer() + // A restart or stop leaves harness sessions running and the next start + // reattaches them; PIPELINES_SHUTDOWN_SESSIONS=stop restores stopping them. + detach, err := shutdownDetaches(os.Getenv(shutdownSessionsEnvKey)) + if err != nil { + log.Fatal(err) + } // A brand-new database starts with the bundled examples; an existing or // already-seeded one is never changed, even if its pipelines were deleted. if seeded, err := server.SeedExamples(ctx, store); err != nil { @@ -62,7 +72,7 @@ func main() { // Each new run gets a bounded one-line title from the configured default // harness; the summariser falls back to the prompt's first line, so title // generation never blocks or fails run creation. - options := server.Options{AllowAuthoring: *allowAuthoring, ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)} + options := server.Options{AllowAuthoring: *allowAuthoring, ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv), DetachSessions: detach} recorded := "" if value, err := store.OpenCodeEndpoint(ctx); err != nil { log.Printf("OpenCode endpoint setting unreadable: %v", err) @@ -117,7 +127,7 @@ func main() { } } if options.Chat == nil { - if conn, err := opencode.OpenChat(ctx, opencode.Executable(), ""); err != nil { + if conn, err := openChat(ctx, detach); err != nil { log.Printf("OpenCode chat unavailable: %v", err) } else if conn.URL == "" { conn.Close() @@ -141,7 +151,11 @@ func main() { chats := server.NewChatManager(ctx, options.Chat, store) if len(chats.Backends()) > 0 { options.Chat = chats - defer chats.Close() + if detach { + defer chats.DetachSessions() + } else { + defer chats.Close() + } } srv := &http.Server{Addr: *addr, Handler: server.HandlerWithOptions(ctx, options, store), ReadHeaderTimeout: 5 * time.Second} // Deferred before WaitRuns so it runs after the runs settle: the owned stage @@ -149,7 +163,16 @@ func main() { defer opencode.StopIdleStageService() defer server.WaitRuns(srv.Handler) go func() { - <-ctx.Done() + select { + case <-signals.Done(): + case <-ctx.Done(): + } + if detach { + // Record the live chats while their rooms still exist. + server.DetachSessions(srv.Handler) + log.Printf("shutdown: harness sessions are left running; the next start reattaches them") + } + cancelServer() shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := srv.Shutdown(shutdownCtx); err != nil { @@ -172,3 +195,36 @@ func main() { log.Fatal(err) } } + +// shutdownSessionsEnvKey selects what a stopping server does with harness +// sessions: "keep" (the default) leaves them running for the next start to +// reattach; "stop" interrupts runs and closes chat clients as before. +const shutdownSessionsEnvKey = "PIPELINES_SHUTDOWN_SESSIONS" + +func shutdownDetaches(value string) (bool, error) { + switch value { + case "", "keep": + return true, nil + case "stop": + return false, nil + } + return false, errors.New(shutdownSessionsEnvKey + " must be keep or stop") +} + +// openChat opens the OpenCode service chat brokers from when no endpoint is +// attached. When sessions are kept across restarts, a V2 installation uses the +// detached Pipelines-owned chat service, which outlives this process; otherwise +// (and for V1) it is a serve child that ends with the server. +func openChat(ctx context.Context, detach bool) (*opencode.Connection, error) { + executable := opencode.Executable() + if detach { + if backend, err := opencode.Detect(ctx, executable); err == nil && backend == "v2" { + directory, err := os.Getwd() + if err != nil { + return nil, err + } + return opencode.EnsureOwnedChatService(ctx, executable, directory) + } + } + return opencode.OpenChat(ctx, executable, "") +} diff --git a/docs/chat-backends.md b/docs/chat-backends.md index 1a5ec940..145b66c8 100644 --- a/docs/chat-backends.md +++ b/docs/chat-backends.md @@ -6,6 +6,45 @@ when the `claude` CLI is installed. The selected backend is stored on the operator-session row and returned as `backend` on every `chat_sessions` row, so a restart does not have to guess from the session id. +## Restarts keep sessions running + +A server restart or stop does not stop chat sessions. Only an operator action +stops one: closing the chat, or Stop on its turn. On shutdown the server records +each chat that still has work on its harness (a turn in flight, a pending +question or approval, or running background tasks) in `chat_live_sessions`. +It then detaches each backend instead of closing it, and never sends an +interrupt. + +On the next start each recorded chat is reattached. The server asks the harness +whether the turn is still running, re-subscribes to its event stream with no +client attached, re-reads pending approvals/questions and background shells, +and holds queued prompts behind the turn. If a turn ended while the server was +down, or its end event was lost with the old stream, the end is folded in from +the harness's history. Clients get a `resync`, the reply notifies once (with +the text the harness recorded), and queued prompts are delivered in order. The +older start-up probe of recently used chats still runs for anything not +recorded, for example after a crash. + +Per backend (the contract is `chatSessionDetacher` in +`internal/server/chat_detach.go`): + +- **OpenCode**: sessions live in the OpenCode service. An attached endpoint + (the operator's CLI-managed service) already outlives the server. Without + one, a V2 server now uses a detached Pipelines-owned chat service + (`opencode serve` in its own session, registered in + `/pipelines/private/opencode-chat-service.json`). The next server + reuses it at the same address and password, and it is not stopped on + shutdown. A V1 serve child still ends with the server. +- **Codex**: threads and turns live in the App Server. Detaching closes the + control connection, and the next server reconciles through `thread/read`. + An approval or question the App Server sent to the old connection is bound to + it, so one still pending at shutdown cannot be answered after the restart. +- **Claude Code**: closed as before until its host implements + `DetachSessions`. + +`PIPELINES_SHUTDOWN_SESSIONS=stop` restores the old shutdown, which closes the +chat clients and records nothing. See [Installation](install.md#restarts-keep-sessions-running). + ## Codex The adapter in `internal/codex` connects to the Codex App Server's Unix diff --git a/docs/harnesses-and-control.md b/docs/harnesses-and-control.md index 88cdbf4a..3d9aa22e 100644 --- a/docs/harnesses-and-control.md +++ b/docs/harnesses-and-control.md @@ -92,7 +92,10 @@ How the stage service behaves: session is active on it. A busy service keeps its old limits until it is idle. - **Lifetime.** It runs detached in its own process session, so a stopping Pipelines service does not take active stage work with it. On server shutdown - it is stopped only when no session is active on it. Otherwise restart + it is stopped only when no session is active on it. A busy service keeps + running, and the next start re-adopts its sessions and continues their runs + (see [Installation](install.md#restarts-keep-sessions-running)). With + `PIPELINES_SHUTDOWN_SESSIONS=stop`, or when the service is gone, restart recovery interrupts the retained sessions through it, as before. - `pipelines debug` is unchanged: it opens a fresh interactive harness conversation, not the stage session. diff --git a/docs/install.md b/docs/install.md index 24603117..ca152a6c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -173,8 +173,8 @@ another installation. | Command | Effect | | --- | --- | | `pipelines start` | start server, web proxy and ntfy; print the startup report | -| `pipelines stop` | stop all three (active runs are interrupted; edits are not rolled back) | -| `pipelines restart` | stop then start, keeping data | +| `pipelines stop` | stop all three; harness sessions keep running and the next start reattaches them (see below) | +| `pipelines restart` | stop then start, keeping data; chat turns and stage sessions continue | | `pipelines doctor` | startup report, health of server and ntfy, OpenCode API check (no inference) | | `pipelines status` | server, host and ntfy publishing health, from the running server | | `pipelines logs` | follow `logs/server.log` | @@ -184,6 +184,50 @@ another installation. | `pipelines service disable` | disable autostart and stop the supervisor | | `pipelines service status` | autostart and supervisor state | +### Restarts keep sessions running + +A server restart, `pipelines stop` and `pipelines update` stop the Pipelines +processes, not the work running on harnesses. On SIGTERM the server leaves +every OpenCode and Codex session running, records what it needs to reattach, +and exits. It does not interrupt or abort anything. On the next start: + +- **Runs**: a stage whose OpenCode V2 session is still alive is re-adopted. The + runner waits for that same session (no new prompt), or reads the result it + reached while the server was down, and then continues the graph. A stage + that had not yet handed work to a harness starts again. A run between stages + continues with its next stage. The stage timeout restarts when the stage is + reattached. Its file-change list is marked partial, because the snapshot + taken before the restart is gone. +- **Chats**: turns, pending questions/approvals, background tasks and queued + prompts are picked up again, and a reply that arrived during the downtime + still notifies. See [Chat backends](chat-backends.md#restarts-keep-sessions-running). + +If the harness is gone, the old behaviour applies and the run is interrupted +with the reason in its diagnostic, for example "Server restarted; the OpenCode +service running stage … was gone". This happens when the OpenCode service +process died, or the session belongs to a Pi or OpenCode V1 process that ended +with the server. + +Only explicit operator actions stop sessions: + +- `pipelines cancel ` (or Cancel in the app) interrupts a run's stage. +- Closing a chat, or Stop on its turn, interrupts that chat. + +To stop everything: cancel the active runs and close or stop the busy chats, +then run `pipelines stop`. The owned OpenCode stage service stops itself at +shutdown when idle. The owned chat service, used only without an attached +endpoint, keeps running and is reused. Stop it by killing the `pid` in +`~/Library/Caches/pipelines/private/opencode-chat-service.json` +(`~/.cache/...` on Linux). To make every shutdown interrupt runs and close chat +clients as before, add `"PIPELINES_SHUTDOWN_SESSIONS": "stop"` to +`opencode_env` in `/pipelines-install.json` and restart. + +Limit on Linux: systemd stops every process in the service's cgroup. The +OpenCode services the server starts (stage service, owned chat service) end +with `pipelines stop`/`restart`, and their runs fall back as described above. +An attached OpenCode endpoint and the Codex App Server are separate and +survive. macOS launchd does not kill them. + The login service is `~/Library/LaunchAgents/net.a-wan.pipelines.plist` on macOS and `~/.config/systemd/user/net.a-wan.pipelines.service` on Linux. It runs `manage.py supervise`, which owns all three processes. This is login diff --git a/internal/codex/chat.go b/internal/codex/chat.go index 35320d78..4e878c5c 100644 --- a/internal/codex/chat.go +++ b/internal/codex/chat.go @@ -176,6 +176,14 @@ func (c *Chat) Close() { _ = c.conn.Close(websocket.StatusNormalClosure, "") } +// DetachSessions is the server-restart shutdown: threads and their turns live +// in the Codex App Server, not in this client, so closing the control +// connection leaves them running; no turn is interrupted. The next server +// process reconnects and reconciles each thread through thread/read. An +// approval or question the App Server sent to this connection is bound to it, +// so one still pending at shutdown is not answerable after the restart. +func (c *Chat) DetachSessions() { c.Close() } + func (c *Chat) readLoop() { for { _, data, err := c.conn.Read(c.ctx) diff --git a/internal/codex/detach_test.go b/internal/codex/detach_test.go new file mode 100644 index 00000000..f47c8ac8 --- /dev/null +++ b/internal/codex/detach_test.go @@ -0,0 +1,48 @@ +package codex + +import ( + "context" + "encoding/json" + "sync" + "sync/atomic" + "testing" +) + +// A server restart detaches from the App Server without interrupting the +// thread's turn; the next server's fresh connection still sees the turn in +// progress, then completed, through the read-only turn list. +func TestDetachSessionsLeavesTurnRunningForNextConnection(t *testing.T) { + var completed atomic.Bool + var mu sync.Mutex + var methods []string + appServer := func(method string, _ json.RawMessage) any { + mu.Lock() + methods = append(methods, method) + mu.Unlock() + status := "inProgress" + if completed.Load() { + status = "completed" + } + return json.RawMessage(`{"data":[{"id":"turn","status":"` + status + `","items":[]}],"nextCursor":null}`) + } + before := modelTestChat(t, appServer) + if running, err := before.ProbeTurn(context.Background(), fixtureThread); err != nil || !running { + t.Fatalf("turn not running before the restart: %v %v", running, err) + } + before.DetachSessions() + after := modelTestChat(t, appServer) + if running, err := after.ProbeTurn(context.Background(), fixtureThread); err != nil || !running { + t.Fatalf("turn did not survive the restart: %v %v", running, err) + } + completed.Store(true) // finished while nobody was watching + if running, err := after.ProbeTurn(context.Background(), fixtureThread); err != nil || running { + t.Fatalf("completed turn still reported running: %v %v", running, err) + } + mu.Lock() + defer mu.Unlock() + for _, method := range methods { + if method == "turn/interrupt" { + t.Fatalf("detaching interrupted the turn: %v", methods) + } + } +} diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index c66f547e..d8b00f58 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -304,6 +304,13 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error { return nil } +// DetachSessions is the server-restart shutdown. Sessions, turns, questions, +// permissions and shells live in the OpenCode service, not in this adapter, so +// there is nothing to stop: the connection's owner decides whether the service +// itself outlives the server (an attached endpoint and the detached owned chat +// service do; a V1 serve child does not). +func (c *Chat) DetachSessions() {} + // Messages reads one session's stored turns. On the owned serve backend it // reuses Connection.Messages, including its read-only OutputFormat readback // fallback, instead of duplicating that path. It maps the verified assistant diff --git a/internal/opencode/chat_owned_test.go b/internal/opencode/chat_owned_test.go new file mode 100644 index 00000000..d764a9c8 --- /dev/null +++ b/internal/opencode/chat_owned_test.go @@ -0,0 +1,64 @@ +package opencode + +import ( + "context" + "errors" + "strings" + "syscall" + "testing" +) + +// The server-spawned chat service is detached and registered: a second server +// process reuses the same running service (sessions survive the restart), and +// it is separate from the stage service. +func TestOwnedChatServiceIsReusedAcrossServerProcesses(t *testing.T) { + executable := fakeOwnedStageEnv(t) + t.Setenv(ChatConfigEnvKey, "") + t.Cleanup(func() { + path, _ := ChatServiceRegistrationPath() + if state, err := readOwnedStageState(path); err == nil { + _ = syscall.Kill(state.PID, syscall.SIGKILL) + } + }) + first, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir()) + if err != nil { + t.Fatal(err) + } + path, _ := ChatServiceRegistrationPath() + stagePath, _ := StageServiceRegistrationPath() + if first.credentialFile != path || path == stagePath || !strings.HasPrefix(first.URL, "http://127.0.0.1:") { + t.Fatalf("chat connection = %+v", first) + } + pid := ownedPID(t, first) + first.Close() // the previous server process exits + second, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir()) + if err != nil || second.URL != first.URL || ownedPID(t, second) != pid { + t.Fatalf("chat service was not reused by the next server: %v", err) + } +} + +// ReferenceAlive tells a live retained service from one whose process is gone, +// without reviving it. +func TestReferenceAliveDoesNotRevive(t *testing.T) { + executable := fakeOwnedStageEnv(t) + c, err := OpenStage(context.Background(), executable, t.TempDir(), "v2") + if err != nil { + t.Fatal(err) + } + reference, err := c.Reference() + if err != nil { + t.Fatal(err) + } + if err := ReferenceAlive(context.Background(), reference); err != nil { + t.Fatalf("live service reported gone: %v", err) + } + pid := ownedPID(t, c) + _ = syscall.Kill(pid, syscall.SIGKILL) + waitGone(t, pid) + if err := ReferenceAlive(context.Background(), reference); !errors.Is(err, ErrServiceNotRunning) { + t.Fatalf("gone service not reported: %v", err) + } + if _, healthy := c.ownedInfo(context.Background()); healthy { + t.Fatal("ReferenceAlive revived the service") + } +} diff --git a/internal/opencode/connection.go b/internal/opencode/connection.go index aa7dcc33..7745f046 100644 --- a/internal/opencode/connection.go +++ b/internal/opencode/connection.go @@ -8,6 +8,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -90,23 +91,65 @@ func (c *Connection) Reference() (string, error) { return f.Name(), nil } -func FromReference(path string) (*Connection, error) { +type connectionReference struct{ Backend, URL, Password, Directory, CredentialFile string } + +func readReference(path string) (connectionReference, error) { + var reference connectionReference info, err := os.Lstat(path) if err != nil { - return nil, err + return reference, err } if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || info.Size() > MaxRequest { - return nil, fmt.Errorf("invalid private OpenCode connection reference") + return reference, fmt.Errorf("invalid private OpenCode connection reference") } if stat, ok := info.Sys().(*syscall.Stat_t); !ok || stat.Uid != uint32(os.Getuid()) { - return nil, fmt.Errorf("OpenCode reference belongs to another user") + return reference, fmt.Errorf("OpenCode reference belongs to another user") } data, err := os.ReadFile(path) if err != nil { - return nil, err + return reference, err } - var reference struct{ Backend, URL, Password, Directory, CredentialFile string } - if err = json.Unmarshal(data, &reference); err != nil { + err = json.Unmarshal(data, &reference) + return reference, err +} + +// ErrServiceNotRunning reports that a retained OpenCode service no longer +// answers: its process is gone, and with it any session turn it was running. +var ErrServiceNotRunning = errors.New("the OpenCode service is not running") + +// ReferenceAlive reports whether the execution service a retained reference +// names is running right now, without starting or reviving it. Restart recovery +// uses it to tell a session that kept running across a Pipelines restart from +// one whose harness process is gone. +func ReferenceAlive(ctx context.Context, path string) error { + reference, err := readReference(path) + if err != nil { + return fmt.Errorf("retained OpenCode reference unreadable: %w", err) + } + if reference.Backend != "v2" { + return fmt.Errorf("the V1 OpenCode stage service belonged to the stopped server process") + } + if !filepath.IsAbs(reference.CredentialFile) { + return fmt.Errorf("invalid V2 service credential reference") + } + c, err := serviceConnection(reference.CredentialFile, reference.Directory) + if err != nil { + return err + } + if c.URL != reference.URL { + return fmt.Errorf("retained OpenCode service endpoint changed") + } + probe, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + if err := c.probeHealth(probe, "v2"); err != nil { + return fmt.Errorf("%w at %s", ErrServiceNotRunning, c.URL) + } + return nil +} + +func FromReference(path string) (*Connection, error) { + reference, err := readReference(path) + if err != nil { return nil, err } if reference.Backend == "v2" { diff --git a/internal/opencode/stage_owned.go b/internal/opencode/stage_owned.go index a041ae78..a8dbc7b5 100644 --- a/internal/opencode/stage_owned.go +++ b/internal/opencode/stage_owned.go @@ -54,14 +54,56 @@ type ownedStageState struct { Identity string `json:"identity"` } -// StageServiceRegistrationPath is the private registration of the owned stage -// service, beside the private connection references. -func StageServiceRegistrationPath() (string, error) { +// ownedService names one Pipelines-owned detached `opencode serve`: the stage +// service, or the chat service a server without an attached endpoint uses. +type ownedService struct { + name string // for logs: "stage" or "chat" + file string // registration file name, without .json +} + +var ( + stageOwnedService = ownedService{name: "stage", file: "opencode-stage-service"} + chatOwnedService = ownedService{name: "chat", file: "opencode-chat-service"} +) + +func (s ownedService) registrationPath() (string, error) { cache, err := os.UserCacheDir() if err != nil { return "", err } - return filepath.Join(cache, "pipelines", "private", "opencode-stage-service.json"), nil + return filepath.Join(cache, "pipelines", "private", s.file+".json"), nil +} + +// StageServiceRegistrationPath is the private registration of the owned stage +// service, beside the private connection references. +func StageServiceRegistrationPath() (string, error) { + return stageOwnedService.registrationPath() +} + +// ChatServiceRegistrationPath is the private registration of the owned chat +// service (see EnsureOwnedChatService). +func ChatServiceRegistrationPath() (string, error) { + return chatOwnedService.registrationPath() +} + +// EnsureOwnedChatService returns a connection to the detached Pipelines-owned +// V2 chat service, starting or reviving it when needed. Unlike a serve child of +// the server, it keeps running when the server stops, so chat turns, questions +// and background shells survive a restart; the next server reuses the same +// endpoint and password from the private registration. Chat-only limits +// (PIPELINES_OPENCODE_CHAT_CONFIG) apply to it alone. +func EnsureOwnedChatService(ctx context.Context, executable, directory string) (*Connection, error) { + overrides, err := chatConfigOverrides() + if err != nil { + return nil, err + } + content := "" + if overrides != nil { + if content, err = configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), overrides); err != nil { + return nil, err + } + } + return ensureOwnedService(ctx, chatOwnedService, content, executable, directory) } func isOwnedStageRegistration(path string) bool { @@ -81,7 +123,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string) if err != nil { return nil, err } - path, err := StageServiceRegistrationPath() + return ensureOwnedService(ctx, stageOwnedService, content, executable, directory) +} + +func ensureOwnedService(ctx context.Context, service ownedService, content, executable, directory string) (*Connection, error) { + path, err := service.registrationPath() if err != nil { return nil, err } @@ -92,7 +138,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string) defer ownedStageMu.Unlock() unlock, err := lockFile(path + ".lock") if err != nil { - return nil, fmt.Errorf("OpenCode stage service lock: %w", err) + return nil, fmt.Errorf("OpenCode %s service lock: %w", service.name, err) } defer unlock() @@ -106,11 +152,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string) case healthy && state.Identity == identity: return ownedStageConnection(path, executable, directory) case healthy && current.ownedIdle(ctx): - log.Printf("OpenCode stage service configuration changed; restarting the idle service") + log.Printf("OpenCode %s service configuration changed; restarting the idle service", service.name) stopOwnedPID(pid) case healthy: // Never interrupt active stage work to apply a new limit. - log.Printf("OpenCode stage service configuration changed; keeping the busy service until it is idle") + log.Printf("OpenCode %s service configuration changed; keeping the busy service until it is idle", service.name) return ownedStageConnection(path, executable, directory) } if u, err := url.Parse(state.URL); err == nil { @@ -135,7 +181,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string) case <-time.After(time.Duration(attempt) * time.Second): } } - started, err := startOwnedStage(ctx, executable, port, state.Password, content) + started, err := startOwnedStage(ctx, service, executable, port, state.Password, content) if err == nil { started.Identity = identity if err := writeOwnedStageState(path, started); err != nil { @@ -197,7 +243,7 @@ func ownedStageIdentity(executable, content string) string { // startOwnedStage starts one detached serve process and waits until it answers // the authenticated V2 info contract. -func startOwnedStage(ctx context.Context, executable string, port int, password, content string) (ownedStageState, error) { +func startOwnedStage(ctx context.Context, service ownedService, executable string, port int, password, content string) (ownedStageState, error) { if port == 0 { free, err := freePort() if err != nil { @@ -214,7 +260,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password, // Its own session: a stopping Pipelines service (and its process group) // does not take active stage work with it. cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - logPath, _ := StageServiceRegistrationPath() + logPath, _ := service.registrationPath() logPath = strings.TrimSuffix(logPath, ".json") + ".log" logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { @@ -224,7 +270,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password, cmd.Stdin = nil if err := cmd.Start(); err != nil { logFile.Close() - return ownedStageState{}, fmt.Errorf("OpenCode stage service could not start: %w", err) + return ownedStageState{}, fmt.Errorf("OpenCode %s service could not start: %w", service.name, err) } logFile.Close() exited := make(chan struct{}) @@ -241,13 +287,13 @@ func startOwnedStage(ctx context.Context, executable string, port int, password, } select { case <-exited: - return ownedStageState{}, fmt.Errorf("OpenCode stage service exited before becoming ready (see %s)", logPath) + return ownedStageState{}, fmt.Errorf("OpenCode %s service exited before becoming ready (see %s)", service.name, logPath) case <-startup.Done(): _ = cmd.Process.Kill() if ctx.Err() != nil { return ownedStageState{}, ctx.Err() } - return ownedStageState{}, fmt.Errorf("OpenCode stage service did not become ready within %s (see %s)", ownedStageStartTimeout, logPath) + return ownedStageState{}, fmt.Errorf("OpenCode %s service did not become ready within %s (see %s)", service.name, ownedStageStartTimeout, logPath) case <-time.After(150 * time.Millisecond): } } diff --git a/internal/pipeline/chat_live_sessions.go b/internal/pipeline/chat_live_sessions.go new file mode 100644 index 00000000..ae6bb973 --- /dev/null +++ b/internal/pipeline/chat_live_sessions.go @@ -0,0 +1,90 @@ +package pipeline + +import ( + "context" + "time" +) + +// ChatLiveSession is one chat a stopping server left live on its harness: a +// turn in flight, an unanswered question or approval, or running background +// tasks. The next server process reattaches each one (and reads back the final +// state of a turn that ended while it was down). +type ChatLiveSession struct { + SessionID string + Backend string + // Turn is true when a turn was in flight; NativeTurnID names it when the + // harness reported one, so a reply is announced once. + Turn bool + NativeTurnID string + // Attention is true when a question/approval was pending or background + // tasks were running. + Attention bool + RecordedAt time.Time +} + +func (s *Store) initChatLiveSessions() error { + _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS chat_live_sessions ( +session_id TEXT PRIMARY KEY, +backend TEXT NOT NULL DEFAULT '', +turn INTEGER NOT NULL DEFAULT 0, +native_turn_id TEXT NOT NULL DEFAULT '', +attention INTEGER NOT NULL DEFAULT 0, +recorded_at INTEGER NOT NULL)`) + return err +} + +// RecordChatLiveSessions replaces the recorded set with sessions. +func (s *Store) RecordChatLiveSessions(ctx context.Context, sessions []ChatLiveSession) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions`); err != nil { + return err + } + for _, session := range sessions { + at := session.RecordedAt + if at.IsZero() { + at = time.Now() + } + if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO chat_live_sessions(session_id,backend,turn,native_turn_id,attention,recorded_at) VALUES(?,?,?,?,?,?)`, + session.SessionID, session.Backend, session.Turn, session.NativeTurnID, session.Attention, at.UnixMilli()); err != nil { + return err + } + } + return tx.Commit() +} + +// TakeChatLiveSessions returns the recorded set and clears it, so each start +// reattaches what the previous process recorded exactly once. +func (s *Store) TakeChatLiveSessions(ctx context.Context) ([]ChatLiveSession, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + rows, err := tx.QueryContext(ctx, `SELECT session_id,backend,turn,native_turn_id,attention,recorded_at FROM chat_live_sessions ORDER BY recorded_at, session_id`) + if err != nil { + return nil, err + } + var sessions []ChatLiveSession + for rows.Next() { + var session ChatLiveSession + var at int64 + if err := rows.Scan(&session.SessionID, &session.Backend, &session.Turn, &session.NativeTurnID, &session.Attention, &at); err != nil { + rows.Close() + return nil, err + } + session.RecordedAt = time.UnixMilli(at) + sessions = append(sessions, session) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions`); err != nil { + return nil, err + } + return sessions, tx.Commit() +} diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index 118ce34a..bb5dbe5a 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -190,18 +190,57 @@ func (s *Store) addColumnIfMissing(table, column, definition string) error { return err } +// RestartStep marks a visit that a stopped server had begun but had not yet +// handed to any harness session as interrupted, so its run can begin the same +// stage again under a new sequence. Nothing was admitted, so nothing replays. +func (s *Store) RestartStep(ctx context.Context, id int64, sequence int, diagnostic string) error { + return affected(s.db.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE run_id=? AND sequence=? AND status='running' AND session_id=''", diagnostic, id, sequence)) +} + func (s *Store) RecoverRuns(ctx context.Context) error { + return s.RecoverRunsKeeping(ctx, nil, nil) +} + +// RecoverRunsKeeping is RecoverRuns for a server that reattaches work across a +// restart: runs in keep are left exactly as the previous process left them (the +// runner continues them), and a run in reasons is interrupted with that reason +// instead of the generic one, so the operator sees why it was not reattached. +func (s *Store) RecoverRunsKeeping(ctx context.Context, keep map[int64]bool, reasons map[int64]string) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() + kept, keptArgs := "", []any{} + for id := range keep { + if kept != "" { + kept += "," + } + kept += "?" + keptArgs = append(keptArgs, id) + } + stepFilter, runFilter := "", "" + if kept != "" { + stepFilter = " AND run_id NOT IN (" + kept + ")" + runFilter = " AND id NOT IN (" + kept + ")" + } // No replay: pending visits never began. A cleanup agent that was still // active is marked interrupted, and its run is interrupted below. - if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)", protocol.StepStatusWaiting); err != nil { + if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)"+stepFilter, append([]any{protocol.StepStatusWaiting}, keptArgs...)...); err != nil { return err } - if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil { + for id, reason := range reasons { + if keep[id] { + continue + } + if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET diagnostic=? WHERE run_id=? AND status='interrupted' AND sequence=(SELECT MAX(sequence) FROM run_steps WHERE run_id=?)", "Server restarted; "+reason, id, id); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", id); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil { return err } // An export checkpoint is a paused handoff, not an interrupted model call. diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 08108798..a9d043ff 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -93,6 +93,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU db.Close() return nil, err } + if err := s.initChatLiveSessions(); err != nil { + db.Close() + return nil, err + } if err := s.initAppNotifications(); err != nil { db.Close() return nil, err diff --git a/internal/server/chat_detach.go b/internal/server/chat_detach.go new file mode 100644 index 00000000..9f1093be --- /dev/null +++ b/internal/server/chat_detach.go @@ -0,0 +1,243 @@ +package server + +import ( + "context" + "log" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// Chat sessions survive a Pipelines restart. Only an operator action (closing +// the chat, or Stop on its turn) stops one. +// +// Per-backend contract. A chat backend whose sessions can outlive the server +// process implements chatSessionDetacher. On shutdown the server calls +// DetachSessions instead of Close: the backend releases its own connection or +// process handles but must not interrupt, abort or kill any session, turn, +// question or background task. On the next start the server reattaches through +// the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still +// running?), Stream (re-subscribe), Permissions/Forms (pending questions and +// approvals), Shells (background tasks) and Messages (read back a turn that +// ended while the server was down). A backend without DetachSessions is closed +// as before, and its sessions end with the server. +// +// OpenCode (an attached or detached owned service) and Codex (its App Server) +// implement it; a Claude Code host that keeps its CLI processes alive can +// implement the same method. +type chatSessionDetacher interface { + DetachSessions() +} + +// chatLiveStore records, on shutdown, the chats left live on their harness so +// the next start reattaches exactly those; *pipeline.Store implements it. +type chatLiveStore interface { + RecordChatLiveSessions(ctx context.Context, sessions []pipeline.ChatLiveSession) error + TakeChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error) +} + +// chatReattachPoll is how often a reattached turn re-asks its harness whether +// it is still running, to fold in an end the re-subscribed stream did not see. +var chatReattachPoll = 2 * time.Second + +// detachSessions records every chat with work still live on its harness: a +// turn in flight, a pending question or approval, or running background tasks. +// It must run before rooms are torn down (before the server context is +// canceled), and it never touches the sessions themselves. +func (h *chatHub) detachSessions(ctx context.Context) error { + if h.liveStore == nil { + return nil + } + h.mu.Lock() + var live []pipeline.ChatLiveSession + for id, room := range h.rooms { + turn := room.running || room.observedTurn + if step := h.turns[id]; step != nil && step.step != nil && step.step.Status == protocol.ChatStepStarted && !step.step.Intermediate { + turn = true + } + attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0 + if turn || attention { + live = append(live, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: room.nativeTurnID, Attention: attention, RecordedAt: h.clock()}) + } + } + h.mu.Unlock() + if len(live) > 0 { + log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live)) + } + return h.liveStore.RecordChatLiveSessions(ctx, live) +} + +// reattachSessions is the start-up half of detachSessions: each recorded chat +// is observed again (stream, pending questions, background tasks) while its +// harness still has work, and a turn that ended while the server was down is +// folded in from the harness's history, so its reply still notifies and its +// queued prompts advance, as if it had been observed live. +func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) { + if h.liveStore == nil || broker == nil { + return + } + live, err := h.liveStore.TakeChatLiveSessions(ctx) + if err != nil { + log.Printf("chat reattach: %v", err) + return + } + for _, session := range live { + if ctx.Err() != nil { + return + } + if !validChatSessionID(session.SessionID) { + continue + } + h.reattachSession(ctx, broker, session) + } +} + +func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) { + id := session.SessionID + running := false + if prober, ok := broker.(chatTurnProber); ok { + probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + busy, err := prober.ProbeTurn(probeCtx, id) + cancel() + if err != nil { + // Unknown is not idle: keep observing rather than guess it ended. + log.Printf("chat reattach probe session=%s failed (%T)", id, err) + busy = session.Turn + } + running = busy + } + if running || session.Attention { + h.observeUnattended(id, broker) + } + if !session.Turn { + return + } + if !running { + h.foldEndedTurn(ctx, broker, session, nil) + return + } + h.mu.Lock() + room := h.rooms[id] + if room == nil { + h.mu.Unlock() + return + } + // The turn the previous process watched is still running: hold the queue + // behind it and remember it as started, so its end notifies as usual. + turn := h.turnLocked(id) + if turn.step == nil || turn.step.Status == protocol.ChatStepEnded { + turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID} + } + if room.nativeTurnID == "" { + room.nativeTurnID = session.NativeTurnID + } + room.observedTurn = true + h.mu.Unlock() + go h.awaitReattachedTurn(room, broker, session) +} + +// awaitReattachedTurn waits for a reattached turn to end. The stream normally +// reports it; when the harness reports the turn idle without the stream having +// said so, the end is folded in from history. +func (h *chatHub) awaitReattachedTurn(room *chatRoom, broker chatBroker, session pipeline.ChatLiveSession) { + prober, _ := broker.(chatTurnProber) + started := h.clock() + ticker := time.NewTicker(chatReattachPoll) + defer ticker.Stop() + for { + select { + case <-room.ctx.Done(): + return + case <-ticker.C: + } + h.mu.Lock() + ended := !room.observedTurn || h.rooms[room.sessionID] != room + h.mu.Unlock() + if ended || prober == nil || h.clock().Sub(started) >= h.unattended.turnLimit { + return + } + probeCtx, cancel := context.WithTimeout(room.ctx, 5*time.Second) + busy, err := prober.ProbeTurn(probeCtx, room.sessionID) + cancel() + if err != nil || busy { + continue + } + h.foldEndedTurn(room.ctx, broker, session, room) + return + } +} + +// foldEndedTurn settles a turn that ended without the hub observing its end: +// attached clients are told to re-read the authoritative history, the room's +// turn slot is released (so queued prompts are delivered) and the reply is +// announced once, with its text read back from the harness. +func (h *chatHub) foldEndedTurn(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession, room *chatRoom) { + id := session.SessionID + readCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + page, err := broker.Messages(readCtx, id, 20, "") + cancel() + if err != nil { + log.Printf("chat reattach history session=%s failed (%T)", id, err) + } + replyID, body := lastReply(page.Messages) + h.mu.Lock() + defer h.mu.Unlock() + if room != nil && h.rooms[id] == room && room.observedTurn { + end := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: room.nativeTurnID}} + h.applyLocked(end) + h.finishTurnLocked(room) + resync := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventResync} + for _, sub := range room.subs { + enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &resync}) + enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &end}) + } + if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) { + room.watching = false + h.releaseRoomLocked(room) + } + } + if replyID == "" || !h.attentionWanted() { + return + } + key := session.NativeTurnID + if key == "" { + key = replyID + } + if !h.claimChatAttentionLocked("reply\x00" + id + "\x00" + key) { + return + } + if body == "" { + body = chatReplyFallbackBody + } + log.Printf("chat session %s: turn finished while the server was restarting; reply read back from history", id) + h.startChatAttention(chatAttention{sessionID: id, reply: true, body: body, id: chatNotificationID(id, "reply", key)}) +} + +// lastReply returns the id and bounded text of the assistant reply that ends +// a history page (messages after the last user message), or "" when the page +// ends with the user's own message. +func lastReply(messages []protocol.ChatMessage) (string, string) { + for i := len(messages) - 1; i >= 0; i-- { + message := messages[i] + if message.Role == protocol.ChatRoleUser { + return "", "" + } + if message.Kind != "" { + continue + } + var text strings.Builder + for _, part := range message.Parts { + if part.Type != protocol.ChatPartText || part.Text == "" { + continue + } + if text.Len() > 0 { + text.WriteString("\n\n") + } + text.WriteString(part.Text) + } + return message.ID, truncateRunes(paragraphs(text.String()), chatReplyBodyRunes) + } + return "", "" +} diff --git a/internal/server/chat_detach_test.go b/internal/server/chat_detach_test.go new file mode 100644 index 00000000..9298272d --- /dev/null +++ b/internal/server/chat_detach_test.go @@ -0,0 +1,203 @@ +package server + +import ( + "context" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// restartChat is a harness fake whose sessions live outside the server, the +// way an OpenCode service or the Codex App Server keeps them: a turn keeps +// running (probe) and its history (Messages) is readable after the server +// that started it is gone. +type restartChat struct { + *observedChat + hmu sync.Mutex + history []protocol.ChatMessage +} + +func newRestartChat() *restartChat { return &restartChat{observedChat: newObservedChat()} } + +func (c *restartChat) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) { + c.hmu.Lock() + defer c.hmu.Unlock() + return protocol.ChatHistoryPage{Messages: append([]protocol.ChatMessage(nil), c.history...)}, nil +} + +func (c *restartChat) setProbe(running bool) { + c.smu.Lock() + c.probe["ses_1"] = running + c.smu.Unlock() +} + +// chatRestartServer is one server process over a shared database. +type chatRestartServer struct { + ctx context.Context + cancel context.CancelFunc + srv *httptest.Server + handler *runHandler +} + +func startChatRestartServer(t *testing.T, broker chatBroker, path string) *chatRestartServer { + t.Helper() + store := openQueueStore(t, path) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + handler := HandlerWithOptions(ctx, Options{Chat: broker, DetachSessions: true}, store).(*runHandler) + s := &chatRestartServer{ctx: ctx, cancel: cancel, srv: httptest.NewServer(handler), handler: handler} + t.Cleanup(s.stop) + return s +} + +// shutdown is the server's SIGTERM path: record live chats, then stop. +func (s *chatRestartServer) shutdown() { + DetachSessions(s.handler) + s.stop() + WaitRuns(s.handler) +} + +func (s *chatRestartServer) stop() { + s.srv.CloseClientConnections() + s.srv.Close() + s.cancel() +} + +func restartChatDB(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "chat.db") + seed := openQueueStore(t, path) + if err := seed.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + if _, err := seed.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil { + t.Fatal(err) + } + seed.Close() + return path +} + +func fastReattach(t *testing.T) { + t.Helper() + fastUnattended(t) + t.Setenv("PIPELINES_NTFY_URL", "") + poll := chatReattachPoll + chatReattachPoll = 20 * time.Millisecond + t.Cleanup(func() { chatReattachPoll = poll }) +} + +// startTurnThenRestart prompts on the first server, holds the turn in flight +// (and optionally queues a second prompt), and shuts that server down. +func startTurnThenRestart(t *testing.T, path string, queueSecond bool) { + t.Helper() + first := newRestartChat() + server := startChatRestartServer(t, first, path) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1") + first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) + waitFor(t, "first prompt", func() bool { return first.promptCount() == 1 }) + if queueSecond { + if ack := sendPrompt(t, server.ctx, conn, "p2", "then ship it", "op-2"); !ack.ChatQueued { + t.Fatalf("second prompt was not queued behind the running turn: %+v", ack) + } + } + server.shutdown() +} + +func replyNotes(t *testing.T, path string) []protocol.AppNotification { + t.Helper() + return chatNotes(t, openQueueStore(t, path), protocol.NotificationChatReply) +} + +// A turn in flight at shutdown that finished while the server was down is +// folded in from the harness's history: its reply notifies once, with the +// text the harness recorded during the downtime. +func TestChatTurnFinishedDuringRestartIsFoldedIn(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + startTurnThenRestart(t, path, false) + second := newRestartChat() + second.setProbe(false) // ended while the server was down + second.history = []protocol.ChatMessage{ + {ID: "msg_u", SessionID: "ses_1", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "build it"}}}, + {ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Built while you were away."}}}, + } + startChatRestartServer(t, second, path) + waitFor(t, "reply folded in from history", func() bool { return len(replyNotes(t, path)) == 1 }) + if notes := replyNotes(t, path); !strings.Contains(notes[0].Message, "Built while you were away.") { + t.Fatalf("reply body not read back from history: %+v", notes[0]) + } +} + +// A turn still running after the restart keeps its queued prompt waiting; +// when the harness reports the turn idle (its end event was lost with the old +// stream), the end is folded in and the queued prompt is delivered. +func TestChatQueuedPromptDeliveredAfterRestart(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + startTurnThenRestart(t, path, true) + second := newRestartChat() + second.setProbe(true) + second.history = []protocol.ChatMessage{{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "First part done."}}}} + startChatRestartServer(t, second, path) + waitFor(t, "reattached stream", func() bool { active, _ := second.streams(); return active >= 1 }) + time.Sleep(2 * chatQueueRecoverySettle) + if second.promptCount() != 0 { + t.Fatalf("queued prompt delivered into a turn still running: %v", second.promptTexts()) + } + second.setProbe(false) + waitFor(t, "queued prompt delivered", func() bool { return second.promptCount() == 1 }) + if texts := second.promptTexts(); texts[0] != "then ship it" { + t.Fatalf("wrong prompt delivered after restart: %v", texts) + } + waitFor(t, "reply notified once", func() bool { return len(replyNotes(t, path)) == 1 }) +} + +// A question pending when the server stopped is observed again after the +// restart with no client attached, and a client that opens the chat sees it. +func TestChatPendingQuestionSurvivesRestart(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + question := protocol.ChatPermissionFrame{SessionID: "ses_1", RequestID: "req_1", Action: "bash", Resources: []string{"make release"}} + first := newRestartChat() + first.perm = []protocol.ChatPermissionFrame{question} + server := startChatRestartServer(t, first, path) + conn := dialChatConn(t, server.ctx, server.srv) + sendPrompt(t, server.ctx, conn, "p1", "release", "op-1") + first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1")) + waitFor(t, "question surfaced", func() bool { return len(server.handler.hub.pendingFrames("ses_1")) == 1 }) + server.shutdown() + + second := newRestartChat() + second.setProbe(true) + second.perm = []protocol.ChatPermissionFrame{question} + restarted := startChatRestartServer(t, second, path) + waitFor(t, "question observed after restart", func() bool { return len(restarted.handler.hub.pendingFrames("ses_1")) == 1 }) + // A client opening the chat after the restart is shown the question and + // its answer reaches the harness. + phone := dialChatConn(t, restarted.ctx, restarted.srv) + writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}) + readUntil(t, restarted.ctx, phone, protocol.ChatSubscribe) + if pending := nextPermission(t, restarted.ctx, phone, "req_1"); pending.Resolved { + t.Fatalf("question resolved by the restart: %+v", pending) + } + writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatReply, ID: "a1", ChatReply: &protocol.ChatReplyRequest{SessionID: "ses_1", RequestID: "req_1", Decision: protocol.ChatDecisionOnce, Client: "Phone"}}) + readUntil(t, restarted.ctx, phone, protocol.ChatReply) + waitFor(t, "answer delivered to the harness", func() bool { return second.replyCount() == 1 }) +} + +// Nothing was in flight: nothing is recorded and nothing is observed. +func TestChatIdleShutdownRecordsNothing(t *testing.T) { + fastReattach(t) + path := restartChatDB(t) + server := startChatRestartServer(t, newRestartChat(), path) + server.shutdown() + live, err := openQueueStore(t, path).TakeChatLiveSessions(context.Background()) + if err != nil || len(live) != 0 { + t.Fatalf("idle shutdown recorded live chats: %+v %v", live, err) + } +} diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 807fea5e..060b32a7 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -54,6 +54,9 @@ type chatHub struct { // queueStore persists prompts queued behind a running turn; nil keeps the // queue in memory only (tests and store-less servers). queueStore chatPromptQueueStore + // liveStore records the chats a stopping server leaves live on their + // harness (chat_detach.go); nil reattaches nothing beyond the start probe. + liveStore chatLiveStore // recentOps/recentOpOrder are the bounded record of accepted prompt // operation ids, so a client retry is acknowledged instead of re-sent. recentOps map[string]struct{} diff --git a/internal/server/chat_manager.go b/internal/server/chat_manager.go index eacd6a76..cbe86fbc 100644 --- a/internal/server/chat_manager.go +++ b/internal/server/chat_manager.go @@ -183,6 +183,28 @@ func (m *chatManager) Close() { closeHarness(claudeClient) } +// DetachSessions is the shutdown counterpart of Close for a server that keeps +// chat sessions running across a restart (see chat_detach.go). Each harness +// that implements chatSessionDetacher releases its handles without stopping a +// session; one that does not is closed as before. The OpenCode connection's +// lifetime belongs to the caller (cmd/pipelines-server), so it is only detached. +func (m *chatManager) DetachSessions() { + m.mu.Lock() + m.closed = true + codexClient, claudeClient := m.codex, m.claude + m.mu.Unlock() + if detacher, ok := m.opencode.(chatSessionDetacher); ok { + detacher.DetachSessions() + } + for _, client := range []chatBroker{codexClient, claudeClient} { + if detacher, ok := client.(chatSessionDetacher); ok { + detacher.DetachSessions() + continue + } + closeHarness(client) + } +} + // Backends lists the harnesses available for a new session. func (m *chatManager) Backends() []string { backends := []string{} diff --git a/internal/server/restart_live_test.go b/internal/server/restart_live_test.go new file mode 100644 index 00000000..74bec5a2 --- /dev/null +++ b/internal/server/restart_live_test.go @@ -0,0 +1,385 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// TestRestartKeepsRealOpenCodeSessions is the opt-in end-to-end restart check +// against the installed OpenCode binary and a real provider: +// +// PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m +// +// It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch +// database and a free loopback port (with only the operator's OpenCode +// provider login copied in), starts a chat turn and a one-stage run that each +// run `sleep 30`, restarts the server mid-turn, and checks both finish +// instead of being aborted. Every scratch process is killed afterwards. +func TestRestartKeepsRealOpenCodeSessions(t *testing.T) { + if os.Getenv("PIPELINES_RESTART_LIVE") != "1" { + t.Skip("set PIPELINES_RESTART_LIVE=1 to run the real-OpenCode restart check") + } + realHome, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + root, err := os.MkdirTemp("", "pipelines-restart-live-") + if err != nil { + t.Fatal(err) + } + t.Logf("scratch root %s", root) + home := filepath.Join(root, "home") + env := scratchEnvironment(t, realHome, home) + binary := filepath.Join(root, "pipelines-server") + build := exec.Command("go", "build", "-o", binary, "./cmd/pipelines-server") + build.Dir = filepath.Join("..", "..") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build: %v\n%s", err, out) + } + // Chat creation activates the manager skill, so install the current + // source's skills into the scratch harness roots. + skills := exec.Command("go", "run", "./cmd/pipelines", "skill", "install") + skills.Dir, skills.Env = filepath.Join("..", ".."), env + if out, err := skills.CombinedOutput(); err != nil { + t.Fatalf("skill install: %v\n%s", err, out) + } + db := filepath.Join(root, "data", "pipelines.db") + if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil { + t.Fatal(err) + } + store, err := pipeline.Open(db) + if err != nil { + t.Fatal(err) + } + if _, err := store.Submit(context.Background(), "restart-live", restartLiveStage); err != nil { + t.Fatal(err) + } + items, _ := store.List(context.Background()) + store.Close() + addr := freeLoopback(t) + logPath := filepath.Join(root, "server.log") + server := startScratchServer(t, binary, db, addr, env, logPath) + t.Cleanup(func() { + server.stop() + killScratchServices(t, home) + }) + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + + chatDir := filepath.Join(root, "chat-work") + runDir := filepath.Join(root, "run-work") + for _, dir := range []string{chatDir, runDir} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + } + client := dialLive(t, ctx, addr) + created := client.request(t, ctx, protocol.Message{Type: protocol.ChatCreate, ID: "c1", ChatCreate: &protocol.ChatCreateRequest{Title: "restart live", Directory: chatDir, OperationID: "create-1"}}, protocol.ChatCreate) + if created.ChatSession == nil { + t.Fatalf("chat_create = %+v", created) + } + sessionID := created.ChatSession.ID + prompted := time.Now() + client.request(t, ctx, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: sessionID, OperationID: "op-1", Client: "restart-live", + Text: "Use your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then reply with exactly the text RESTART-CHAT-OK and nothing else."}}, protocol.ChatPrompt) + started := client.request(t, ctx, protocol.Message{Type: protocol.RunPipeline, ID: "r1", PipelineID: items[0].ID, Version: 1, RequestKey: "restart-live", Workspace: runDir, Input: map[string]any{"prompt": "restart live check"}}, protocol.RunStarted) + if started.RunDetail == nil { + t.Fatalf("run_pipeline = %+v", started) + } + runID := started.RunDetail.ID + // Wait until both harness sessions are executing the sleep. + waitLive(t, ctx, "chat shell running", func() bool { return chatShellStatus(t, ctx, addr, sessionID) == "running" }) + var stageSession string + waitLive(t, ctx, "stage session admitted", func() bool { + d := getLiveRun(t, ctx, addr, runID) + if len(d.Steps) == 1 && d.Steps[0].SessionID != "" { + stageSession = d.Steps[0].SessionID + return true + } + return false + }) + time.Sleep(8 * time.Second) // the stage's shell is inside its sleep + client.close() + + t.Log("restarting the scratch server mid-turn") + server.stop() + server = startScratchServer(t, binary, db, addr, env, logPath) + + var reply string + waitLiveFor(t, ctx, 5*time.Minute, "chat reply after restart", func() bool { + reply = chatReply(t, ctx, addr, sessionID) + return strings.Contains(reply, "RESTART-CHAT-OK") + }) + if status := chatShellStatus(t, ctx, addr, sessionID); status != "completed" { + t.Fatalf("chat shell was not completed after the restart: %q", status) + } + if elapsed := time.Since(prompted); elapsed < 30*time.Second { + t.Fatalf("chat reply arrived before the sleep could finish (%s): the turn was cut short", elapsed) + } + var final *protocol.RunDetail + waitLiveFor(t, ctx, 6*time.Minute, "run finished after restart", func() bool { + final = getLiveRun(t, ctx, addr, runID) + return final.FinishedAt != "" + }) + if final.Status != "succeeded" || len(final.Steps) != 1 || final.Steps[0].Status != "completed" || final.Steps[0].SessionID != stageSession { + t.Fatalf("run did not continue on its own session: status=%s diagnostic=%q steps=%+v", final.Status, final.Diagnostic, final.Steps) + } + logData, _ := os.ReadFile(logPath) + for _, want := range []string{"leaving 1 chat session(s) running", "left running for the next start to reattach", "reattached stage one"} { + if !bytes.Contains(logData, []byte(want)) { + t.Errorf("server log lacks %q", want) + } + } + t.Logf("chat reply %q; run %d %s on session %s", reply, runID, final.Status, stageSession) +} + +const restartLiveStage = "# stage one\n\n```prompt\nUse your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then write the turn output: a short body saying RESTART-STAGE-OK, with the outcome success.\n```\n\n```tools\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\ntimeout=15m\n```\n" + +// scratchEnvironment is the server's environment: a private HOME/XDG tree with +// only the operator's OpenCode provider login and a minimal configuration. +func scratchEnvironment(t *testing.T, realHome, home string) []string { + t.Helper() + config, data, cache := filepath.Join(home, ".config"), filepath.Join(home, ".local", "share"), filepath.Join(home, ".cache") + for _, dir := range []string{filepath.Join(config, "opencode"), filepath.Join(data, "opencode"), cache} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + } + auth, err := os.ReadFile(filepath.Join(realHome, ".local", "share", "opencode", "auth.json")) + if err != nil { + t.Skipf("no OpenCode login to copy: %v", err) + } + if err := os.WriteFile(filepath.Join(data, "opencode", "auth.json"), auth, 0600); err != nil { + t.Fatal(err) + } + // A cold model catalog can leave the first service unable to resolve the + // model; seed the public catalog cache (not a credential). + if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil { + if err := os.MkdirAll(filepath.Join(cache, "opencode"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cache, "opencode", "models.json"), catalog, 0600); err != nil { + t.Fatal(err) + } + } + settings := `{"$schema":"https://opencode.ai/config.json","model":"opencode-go/deepseek-v4.1-flash","permissions":[{"action":"shell","resource":"*","effect":"allow"}]}` + if err := os.WriteFile(filepath.Join(config, "opencode", "opencode.json"), []byte(settings), 0600); err != nil { + t.Fatal(err) + } + var env []string + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + switch { + case key == "HOME", strings.HasPrefix(key, "XDG_"), strings.HasPrefix(key, "PIPELINES_"), strings.HasPrefix(key, "OPENCODE_"): + continue + } + env = append(env, entry) + } + return append(env, "HOME="+home, "XDG_CONFIG_HOME="+config, "XDG_DATA_HOME="+data, "XDG_CACHE_HOME="+cache, "XDG_STATE_HOME="+filepath.Join(home, ".local", "state"), + "PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off") +} + +func opencodePath() string { + if path, err := exec.LookPath("opencode"); err == nil { + return path + } + return "/opt/homebrew/bin/opencode" +} + +type scratchServer struct { + t *testing.T + cmd *exec.Cmd +} + +func startScratchServer(t *testing.T, binary, db, addr string, env []string, logPath string) *scratchServer { + t.Helper() + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(binary, "-db", db, "-addr", addr, "-mdns=false") + cmd.Env, cmd.Stdout, cmd.Stderr = env, logFile, logFile + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + logFile.Close() + s := &scratchServer{t: t, cmd: cmd} + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + if resp, err := http.Get("http://" + addr + "/healthz"); err == nil { + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return s + } + } + time.Sleep(200 * time.Millisecond) + } + s.stop() + data, _ := os.ReadFile(logPath) + t.Fatalf("scratch server did not become healthy:\n%s", data) + return nil +} + +// stop is a service stop/restart: SIGTERM, then wait for a clean exit. +func (s *scratchServer) stop() { + if s == nil || s.cmd == nil || s.cmd.Process == nil || s.cmd.ProcessState != nil { + return + } + _ = s.cmd.Process.Signal(syscall.SIGTERM) + done := make(chan struct{}) + go func() { _ = s.cmd.Wait(); close(done) }() + select { + case <-done: + case <-time.After(60 * time.Second): + _ = s.cmd.Process.Kill() + <-done + s.t.Error("scratch server did not stop within 60s of SIGTERM") + } +} + +// killScratchServices stops the detached OpenCode services the scratch server +// owned (their pids are in its private registrations). +func killScratchServices(t *testing.T, home string) { + for _, name := range []string{"opencode-stage-service.json", "opencode-chat-service.json"} { + data, err := os.ReadFile(filepath.Join(home, "Library", "Caches", "pipelines", "private", name)) + if err != nil { + data, err = os.ReadFile(filepath.Join(home, ".cache", "pipelines", "private", name)) + } + if err != nil { + continue + } + var state struct{ PID int } + if json.Unmarshal(data, &state) == nil && state.PID > 0 { + _ = syscall.Kill(state.PID, syscall.SIGTERM) + time.Sleep(time.Second) + _ = syscall.Kill(state.PID, syscall.SIGKILL) + t.Logf("stopped scratch %s (pid %d)", strings.TrimSuffix(name, ".json"), state.PID) + } + } +} + +func freeLoopback(t *testing.T) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + return "127.0.0.1:" + strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) +} + +type liveClient struct{ conn *websocket.Conn } + +func dialLive(t *testing.T, ctx context.Context, addr string) *liveClient { + t.Helper() + conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/connect", nil) + if err != nil { + t.Fatal(err) + } + conn.SetReadLimit(16 << 20) + return &liveClient{conn: conn} +} + +func (c *liveClient) close() { c.conn.CloseNow() } + +func (c *liveClient) request(t *testing.T, ctx context.Context, msg protocol.Message, want string) protocol.Message { + t.Helper() + if err := wsjson.Write(ctx, c.conn, msg); err != nil { + t.Fatal(err) + } + read, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + for { + var reply protocol.Message + if err := wsjson.Read(read, c.conn, &reply); err != nil { + t.Fatalf("waiting for %s: %v", want, err) + } + if reply.ID != msg.ID { + continue + } + if reply.Type == protocol.Error { + t.Fatalf("%s refused: %s", msg.Type, reply.Error) + } + if reply.Type == want { + return reply + } + } +} + +func liveQuery(t *testing.T, ctx context.Context, addr string, msg protocol.Message, want string) protocol.Message { + t.Helper() + c := dialLive(t, ctx, addr) + defer c.close() + return c.request(t, ctx, msg, want) +} + +func getLiveRun(t *testing.T, ctx context.Context, addr string, id int64) *protocol.RunDetail { + t.Helper() + reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.GetRun, ID: "g", RunID: id}, protocol.Run) + if reply.RunDetail == nil { + t.Fatalf("get_run = %+v", reply) + } + return reply.RunDetail +} + +func liveMessages(t *testing.T, ctx context.Context, addr, session string) []protocol.ChatMessage { + t.Helper() + reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.ChatMessages, ID: "m", ChatQuery: &protocol.ChatQuery{SessionID: session, Limit: 50}}, protocol.ChatMessages) + if reply.ChatHistory != nil { + return reply.ChatHistory.Messages + } + return reply.ChatMessages +} + +// chatShellStatus reports the status of the newest shell tool call. +func chatShellStatus(t *testing.T, ctx context.Context, addr, session string) string { + status := "" + for _, message := range liveMessages(t, ctx, addr, session) { + for _, part := range message.Parts { + if part.Type == protocol.ChatPartTool && part.State != nil && strings.Contains(strings.ToLower(part.Name+string(part.State.Input)), "sleep") { + status = part.State.Status + } + } + } + return status +} + +func chatReply(t *testing.T, ctx context.Context, addr, session string) string { + _, text := lastReply(liveMessages(t, ctx, addr, session)) + return text +} + +func waitLive(t *testing.T, ctx context.Context, what string, check func() bool) { + t.Helper() + waitLiveFor(t, ctx, 3*time.Minute, what, check) +} + +func waitLiveFor(t *testing.T, ctx context.Context, limit time.Duration, what string, check func() bool) { + t.Helper() + deadline := time.Now().Add(limit) + for time.Now().Before(deadline) && ctx.Err() == nil { + if check() { + return + } + time.Sleep(2 * time.Second) + } + t.Fatal(fmt.Sprintf("timed out waiting for %s", what)) +} diff --git a/internal/server/run_reattach.go b/internal/server/run_reattach.go new file mode 100644 index 00000000..bc7d90d7 --- /dev/null +++ b/internal/server/run_reattach.go @@ -0,0 +1,220 @@ +package server + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "git.a-wan.net/alan/pipelines/internal/opencode" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// Runs survive a server restart. With detach enabled (the installed server), +// a stopping server cancels each run's context with stage.ErrServerShutdown: +// stage clients leave their admitted harness session running and owned, and the +// loop returns without settling anything. The next process plans, before it +// serves requests, how to continue every run it finds in flight: +// +// - a running visit with a live OpenCode V2 session is re-adopted: the loop +// re-enters that visit, attaches to the session, waits for it to settle (or +// reads the result it reached while the server was down) and continues the +// graph as if it had watched it; +// - a visit begun but never handed to a harness is marked interrupted and the +// same stage starts again (nothing was admitted, so nothing replays); +// - a run between visits continues with its next stage; +// - a session whose harness process is gone, or that belongs to a harness the +// server owned as a child (Pi, OpenCode V1), falls back to the previous +// behaviour: the run is interrupted, with the reason in its diagnostic. +// +// Only an operator cancel (cancel_run) stops a stage session. + +type reattachKind int + +const ( + reattachAdopt reattachKind = iota + reattachRestart + reattachBetween +) + +// reattachPlan is one run start-up recovery kept for reattachRuns. +type reattachPlan struct { + runID int64 + kind reattachKind + // step is the running visit (adopt/restart) or the last completed one. + step protocol.RunStep + owner pipeline.SessionOwner +} + +// adoptedVisit is the visit a runLoop re-adopts instead of starting. +type adoptedVisit struct { + sequence int + owner pipeline.SessionOwner +} + +// stageServiceAlive reports whether a retained stage session's harness service +// is still running. It is a seam for tests. +var stageServiceAlive = func(ctx context.Context, owner pipeline.SessionOwner) error { + return opencode.ReferenceAlive(ctx, owner.Reference) +} + +// interruptRecoveredSession stops a previously active session that start-up +// recovery does not re-adopt. It is a seam for tests. +var interruptRecoveredSession = func(ctx context.Context, owner pipeline.SessionOwner) error { + if owner.Backend == "pi" { + return stage.InterruptPi(ctx, owner.Reference) + } + return (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(ctx, owner.ID) +} + +// runContext derives one run's execution context. With detach, a server +// shutdown cancels it with stage.ErrServerShutdown as the cause, while the +// returned cancel (an operator cancel or the run's own end) uses the ordinary +// context.Canceled, so stage clients can tell the two apart. +func (r *runner) runContext() (context.Context, context.CancelFunc) { + if !r.detach { + return context.WithCancel(r.ctx) + } + ctx, cancel := context.WithCancelCause(context.WithoutCancel(r.ctx)) + stop := context.AfterFunc(r.ctx, func() { cancel(stage.ErrServerShutdown) }) + return ctx, func() { stop(); cancel(context.Canceled) } +} + +// planReattach decides, per in-flight run, whether start-up continues it. +// Runs it does not plan keep today's recovery (interrupted); reasons carries the +// explanation for the ones that could not be reattached. +// gone lists runs whose session's service is not running: there is nothing +// left to interrupt, so recovery only clears their ownership. +func planReattach(ctx context.Context, s *pipeline.Store) ([]reattachPlan, map[int64]string, map[int64]bool, error) { + reasons, gone := map[int64]string{}, map[int64]bool{} + rows, err := s.ListRuns(ctx) + if err != nil { + return nil, nil, nil, err + } + var plans []reattachPlan + for _, row := range rows { + if row.Status != protocol.RunStatusRunning || row.CancelRequested || row.CleanupUnconfirmed { + continue + } + detail, err := s.GetRun(ctx, row.ID) + if err != nil { + return nil, nil, nil, err + } + if len(detail.Steps) == 0 { + continue + } + last := detail.Steps[0] + for _, step := range detail.Steps { + if step.Sequence > last.Sequence { + last = step + } + } + switch last.Status { + case protocol.StepStatusRunning: + if last.SessionID == "" { + plans = append(plans, reattachPlan{runID: row.ID, kind: reattachRestart, step: last}) + continue + } + owner, err := s.RunSessionOwner(ctx, row.ID, last.Sequence) + if err != nil { + return nil, nil, nil, err + } + owner.RunID = row.ID + if owner.Backend != "v2" { + name := "OpenCode V1" + if owner.Backend == "pi" { + name = "Pi" + } + reasons[row.ID] = fmt.Sprintf("%s stage sessions belong to the stopped server process, so stage %s could not be reattached", name, last.Stage) + continue + } + probe, cancel := context.WithTimeout(ctx, 5*time.Second) + aliveErr := stageServiceAlive(probe, owner) + cancel() + if aliveErr != nil { + gone[row.ID] = errors.Is(aliveErr, opencode.ErrServiceNotRunning) + reasons[row.ID] = fmt.Sprintf("the OpenCode service running stage %s was gone (%v), so its session could not be reattached", last.Stage, aliveErr) + continue + } + plans = append(plans, reattachPlan{runID: row.ID, kind: reattachAdopt, step: last, owner: owner}) + case "completed": + plans = append(plans, reattachPlan{runID: row.ID, kind: reattachBetween, step: last}) + } + } + return plans, reasons, gone, nil +} + +// reattachRuns continues every run start-up recovery kept. It is called once, +// after the runner's collaborators are wired and before requests are served. +func (r *runner) reattachRuns() { + plans := r.reattach + r.reattach = nil + for _, plan := range plans { + r.reattachRun(plan) + } +} + +func (r *runner) reattachRun(plan reattachPlan) { + id := plan.runID + ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second) + defer cancel() + if plan.kind == reattachRestart { + if err := r.store.RestartStep(ctx, id, plan.step.Sequence, "Server restarted before this stage handed work to a harness; the stage was started again"); err != nil { + log.Printf("run %d: restart stage %s after server restart: %v", id, plan.step.Stage, err) + } + } + detail, err := r.store.GetRun(ctx, id) + if err != nil { + log.Printf("run %d: read for reattach: %v", id, err) + return + } + r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) + workspace := detail.Workspace + request := protocol.Message{WritePaths: detail.WritePaths, AllowTools: detail.AllowTools, Model: detail.Model, StageModels: detail.StageModels} + r.mu.Lock() + defer r.mu.Unlock() + switch plan.kind { + case reattachBetween: + log.Printf("run %d: continuing after stage %s following a server restart", id, plan.step.Stage) + r.launch(id, workspace, func(ctx context.Context) { r.resume(ctx, id, detail, workspace, plan.step) }) + default: + state := rebuildAdoptState(detail, plan) + if plan.kind == reattachAdopt { + log.Printf("run %d: reattached stage %s #%d session %s after a server restart", id, plan.step.Stage, plan.step.Sequence, plan.owner.ID) + } else { + log.Printf("run %d: starting stage %s again after a server restart", id, plan.step.Stage) + } + r.launch(id, workspace, func(ctx context.Context) { + r.runLoop(ctx, id, detail.Graph, detail.Name, workspace, request, state) + }) + } +} + +// rebuildAdoptState restores the loop state at the planned visit: memory and +// visit counts from every other persisted step, the handoff input from the +// visit before it, and (for an adopted visit) the session to re-adopt. +func rebuildAdoptState(detail *protocol.RunDetail, plan reattachPlan) *runState { + decided := protocol.RunStep{Target: plan.step.Stage, Output: detail.Input} + previous := 0 + for _, step := range detail.Steps { + if step.Sequence < plan.step.Sequence && step.Sequence > previous { + previous = step.Sequence + decided = step + decided.Target = plan.step.Stage + } + } + state := rebuildRunState(detail, decided) + state.key = plan.step.Stage + // The interrupted or adopted visit was counted; the loop counts it again. + if state.visits[plan.step.Stage] > 0 { + state.visits[plan.step.Stage]-- + } + if plan.kind == reattachAdopt { + state.nextSequence = plan.step.Sequence + state.adopt = &adoptedVisit{sequence: plan.step.Sequence, owner: plan.owner} + } + return state +} diff --git a/internal/server/run_reattach_test.go b/internal/server/run_reattach_test.go new file mode 100644 index 00000000..047170ca --- /dev/null +++ b/internal/server/run_reattach_test.go @@ -0,0 +1,262 @@ +package server + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/opencode" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// restartFixture runs a two-stage pipeline on a detaching runner, stops that +// "server" while stage one is in flight, and reopens the store for a second +// runner, the way a restart does. +type restartFixture struct { + path, root, workspace string + runID int64 +} + +// stopMidStage starts the run and shuts the first runner down while stage one +// is executing. firstStage is the first process's executor for stage one; it +// must block until its context ends. +func stopMidStage(t *testing.T, firstStage func(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error)) restartFixture { + t.Helper() + f := restartFixture{path: filepath.Join(t.TempDir(), "restart.db"), root: t.TempDir(), workspace: t.TempDir()} + s, err := pipeline.Open(f.path) + if err != nil { + t.Fatal(err) + } + ctx, shutdown := context.WithCancel(context.Background()) + if _, err := s.Submit(ctx, "restart", pipeline.Template([]string{"one", "two"}, true)); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + r, err := newRunnerReattaching(ctx, s, f.root, true) + if err != nil { + t.Fatal(err) + } + r.exportSession = validGateExport + entered := make(chan struct{}) + var id atomic.Int64 + r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + if def.Key != "one" { + t.Errorf("the first process reached stage %s", def.Key) + return stage.Result{}, fmt.Errorf("unexpected stage") + } + for id.Load() == 0 { + time.Sleep(time.Millisecond) + } + close(entered) + return firstStage(ctx, s, id.Load()) + } + d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "restart", Workspace: f.workspace, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + id.Store(d.ID) + f.runID = d.ID + <-entered + shutdown() // SIGTERM + r.wg.Wait() + r.schedulerWG.Wait() + persisted, err := s.GetRun(context.Background(), f.runID) + if err != nil { + t.Fatal(err) + } + if persisted.Status != protocol.RunStatusRunning || len(persisted.Steps) != 1 || persisted.Steps[0].Status != protocol.StepStatusRunning { + t.Fatalf("shutdown settled the run instead of leaving it in flight: %+v", persisted) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + return f +} + +// restart opens the second process's runner on the same database. +func (f restartFixture) restart(t *testing.T, execute func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error)) (*runner, *pipeline.Store) { + t.Helper() + s, err := pipeline.Open(f.path) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + r, err := newRunnerReattaching(ctx, s, f.root, true) + if err != nil { + cancel() + s.Close() + t.Fatal(err) + } + t.Cleanup(func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() }) + r.exportSession = validGateExport + r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + return execute(ctx, s, def) + } + return r, s +} + +func seamRestartRecovery(t *testing.T, alive error) *atomic.Int32 { + t.Helper() + interrupts := &atomic.Int32{} + previousAlive, previousInterrupt := stageServiceAlive, interruptRecoveredSession + stageServiceAlive = func(context.Context, pipeline.SessionOwner) error { return alive } + interruptRecoveredSession = func(context.Context, pipeline.SessionOwner) error { interrupts.Add(1); return nil } + t.Cleanup(func() { stageServiceAlive, interruptRecoveredSession = previousAlive, previousInterrupt }) + return interrupts +} + +// ownedLiveStage is a first-process stage one that admitted session +// ses_live (recorded like a real client does) and is still running at shutdown. +func ownedLiveStage(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error) { + if err := s.RunSession(context.Background(), id, 1, "ses_live", true, "v2", "ref-live"); err != nil { + return stage.Result{}, err + } + <-ctx.Done() + if stage.Detached(ctx) { + return stage.Result{}, stage.ErrDetached + } + return stage.Result{}, ctx.Err() +} + +// TestStageRunContinuesToNextStageAfterRestart: the stage in flight at a server +// restart is re-adopted (never re-prompted, never interrupted), its result is +// folded in, and the run continues with the next stage to success. +func TestStageRunContinuesToNextStageAfterRestart(t *testing.T) { + interrupts := seamRestartRecovery(t, nil) + f := stopMidStage(t, ownedLiveStage) + var resumed atomic.Value + var secondRan atomic.Int32 + r, s := f.restart(t, func(ctx context.Context, s *pipeline.Store, def stage.Definition) (stage.Result, error) { + if def.Key == "one" { + resumed.Store(stage.TakeResumeSession(ctx)) + // The real client releases ownership once the session settled. + if err := s.RunSession(context.Background(), f.runID, 1, "ses_live", false, "v2", "ref-live"); err != nil { + return stage.Result{}, err + } + return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "finished while the server was down"}}, nil + } + secondRan.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next stage"}}, nil + }) + if interrupts.Load() != 0 { + t.Fatal("start-up recovery interrupted a session that survived the restart") + } + r.reattachRuns() + r.wg.Wait() + d, err := s.GetRun(context.Background(), f.runID) + if err != nil { + t.Fatal(err) + } + if got, _ := resumed.Load().(string); got != "ses_live" { + t.Fatalf("stage one was not re-adopted from its session: %q", got) + } + if d.Status != "succeeded" || len(d.Steps) != 2 || secondRan.Load() != 1 { + t.Fatalf("run did not continue to the next stage: %+v", d) + } + one := d.Steps[0] + if one.Sequence != 1 || one.Status != "completed" || one.Output["markdown"] != "finished while the server was down" { + t.Fatalf("adopted visit not folded in: %+v", one) + } + if one.FileChanges == nil || one.FileChanges.Status != "partial" || !strings.Contains(one.FileChanges.Diagnostic, "restart") { + t.Fatalf("adopted visit claims a complete change capture: %+v", one.FileChanges) + } +} + +// TestStageRunWithHarnessGoneFallsBackWithReason: when the harness process +// is gone after the restart, the run is interrupted as before, naming why. +func TestStageRunWithHarnessGoneFallsBackWithReason(t *testing.T) { + interrupts := seamRestartRecovery(t, fmt.Errorf("%w at http://127.0.0.1:1", opencode.ErrServiceNotRunning)) + f := stopMidStage(t, ownedLiveStage) + r, s := f.restart(t, func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error) { + t.Error("a run whose harness was gone executed again") + return stage.Result{}, nil + }) + r.reattachRuns() + r.wg.Wait() + if interrupts.Load() != 0 { + t.Fatal("recovery tried to interrupt a session on a service that is gone") + } + d, err := s.GetRun(context.Background(), f.runID) + if err != nil { + t.Fatal(err) + } + if d.Status != "interrupted" || !strings.Contains(d.Diagnostic, "was gone") || !strings.Contains(d.Steps[0].Diagnostic, "was gone") { + t.Fatalf("harness-gone fallback lacks a clear reason: status=%s run=%q step=%q", d.Status, d.Diagnostic, d.Steps[0].Diagnostic) + } + if owners, _ := s.ActiveSessionOwners(context.Background()); len(owners) != 0 { + t.Fatalf("a gone session is still recorded as active: %+v", owners) + } +} + +// TestStageNotYetAdmittedStartsAgainAfterRestart: a visit the stopped server +// had begun but not handed to any session is started again, nothing replayed. +func TestStageNotYetAdmittedStartsAgainAfterRestart(t *testing.T) { + seamRestartRecovery(t, nil) + f := stopMidStage(t, func(ctx context.Context, _ *pipeline.Store, _ int64) (stage.Result, error) { + <-ctx.Done() + return stage.Result{}, ctx.Err() + }) + var resumed atomic.Value + resumed.Store("") + r, s := f.restart(t, func(ctx context.Context, _ *pipeline.Store, def stage.Definition) (stage.Result, error) { + if def.Key == "one" { + resumed.Store(stage.TakeResumeSession(ctx)) + } + return stage.Result{Outcome: "success", Output: stage.Values{"markdown": def.Key}}, nil + }) + r.reattachRuns() + r.wg.Wait() + d, _ := s.GetRun(context.Background(), f.runID) + if d.Status != "succeeded" || len(d.Steps) != 3 || d.Steps[0].Status != "interrupted" || d.Steps[1].Stage != "one" || d.Steps[1].Status != "completed" { + t.Fatalf("unadmitted stage was not started again: %+v", d) + } + if got := resumed.Load().(string); got != "" { + t.Fatalf("a fresh visit was treated as a re-adoption of %q", got) + } +} + +// TestOperatorCancelStillStopsDetachingRun: only shutdown detaches; an +// operator cancel on a detaching server still settles the run as canceled. +func TestOperatorCancelStillStopsDetachingRun(t *testing.T) { + s, err := pipeline.Open(filepath.Join(t.TempDir(), "cancel.db")) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + if _, err := s.Submit(ctx, "cancel", pipeline.Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + r, err := newRunnerReattaching(ctx, s, t.TempDir(), true) + if err != nil { + t.Fatal(err) + } + defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() }() + entered := make(chan struct{}) + detached := &atomic.Bool{} + r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { + close(entered) + <-ctx.Done() + detached.Store(stage.Detached(ctx)) + return stage.Result{}, ctx.Err() + } + d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "cancel", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + <-entered + if err := r.cancel(d.ID); err != nil { + t.Fatal(err) + } + r.wg.Wait() + final, _ := s.GetRun(context.Background(), d.ID) + if detached.Load() || final.Status != "canceled" { + t.Fatalf("operator cancel did not stop the run: detached=%v %+v", detached.Load(), final) + } +} diff --git a/internal/server/runner.go b/internal/server/runner.go index e67e888f..20464db2 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -84,6 +84,10 @@ type runner struct { // sizeKick wakes the background run-size backfill (sizeLoop); nil when the // loop is not running (a runner built directly in a test). sizeKick chan struct{} + // detach leaves harness sessions running when the server stops; reattach + // holds the runs start-up recovery kept for reattachRuns to continue. + detach bool + reattach []reattachPlan } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { @@ -96,6 +100,14 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { // starts and is never written afterwards, so those goroutines read it without // locking. func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) { + return newRunnerReattaching(ctx, s, artifactsRoot, false) +} + +// newRunnerReattaching builds the runner. With detach set, a server shutdown +// leaves stage sessions running (see stage.ErrServerShutdown) and start-up +// recovery re-adopts the runs a previous process left in flight instead of +// interrupting them; reattachRuns then continues them. +func newRunnerReattaching(ctx context.Context, s *pipeline.Store, artifactsRoot string, detach bool) (*runner, error) { if err := s.AcquireRunnerLock(); err != nil { return nil, err } @@ -109,19 +121,30 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) ( if err = s.BindLegacyScheduleWorkspace(ctx, workspace); err != nil { return nil, err } + var plans []reattachPlan + keep, reasons, gone := map[int64]bool{}, map[int64]string{}, map[int64]bool{} + if detach { + if plans, reasons, gone, err = planReattach(ctx, s); err != nil { + return nil, err + } + for _, plan := range plans { + keep[plan.runID] = true + } + } sessions, err := s.ActiveSessionOwners(ctx) if err != nil { return nil, err } for _, owner := range sessions { - stop, cancel := context.WithTimeout(ctx, 20*time.Second) - var err error - if owner.Backend == "pi" { - err = stage.InterruptPi(stop, owner.Reference) - } else { - err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID) + if keep[owner.RunID] { + continue // Re-adopted below: never interrupt work that survived. + } + var err error + if !gone[owner.RunID] { + stop, cancel := context.WithTimeout(ctx, 20*time.Second) + err = interruptRecoveredSession(stop, owner) + cancel() } - cancel() if err != nil { return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } @@ -129,7 +152,7 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) ( return nil, err } } - if err := s.RecoverRuns(ctx); err != nil { + if err := s.RecoverRunsKeeping(ctx, keep, reasons); err != nil { return nil, err } root := artifactsRoot @@ -146,7 +169,7 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) ( active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace} } } - r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession} + r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession, detach: detach, reattach: plans} if err := r.startSessionExportJobs(); err != nil { return nil, err } @@ -341,7 +364,7 @@ func (r *runner) startWithSchedule(request protocol.Message, origin *pipeline.Sc // the registration when it returns. A fresh run and a resumed decision both go // through it, so registration and teardown live once. Callers must hold r.mu. func (r *runner) launch(id int64, workspace string, fn func(context.Context)) { - ctx, cancel := context.WithCancel(r.ctx) + ctx, cancel := r.runContext() generation := new(byte) r.active[id] = activeRun{cancel: cancel, workspace: workspace, generation: generation} r.wg.Add(1) @@ -980,6 +1003,9 @@ type runState struct { // have been resolved. It is intentionally not part of run input or memory. retryOverride stage.Values priorBudgets map[string]stage.BudgetSnapshot + // adopt names the visit to re-adopt from a previous server process instead + // of starting it; it is consumed by the loop's first iteration. + adopt *adoptedVisit } func newRunState(key string, input stage.Values) *runState { @@ -1528,6 +1554,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na finish := func(status, outcome, diagnostic string) { r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input) } + // canceled settles an operator cancel. A server shutdown is not a cancel: + // the run is left exactly as persisted for the next process to continue. + canceled := func() { + if stage.Detached(ctx) { + log.Printf("run %d: server stopping; run left in flight for the next start to continue", id) + return + } + finish("canceled", "", ctx.Err().Error()) + } defer func() { if recovered := recover(); recovered != nil { finish("failed", "error", "stage execution panicked") @@ -1554,7 +1589,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } for sequence := state.nextSequence; sequence <= 30; sequence++ { if ctx.Err() != nil { - finish("canceled", "", ctx.Err().Error()) + canceled() return } key := state.key @@ -1600,8 +1635,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na return } } + // adopted is the visit a previous server process began and left running + // across its restart: its step row, turn files and session already exist. + adopted := state.adopt + if adopted != nil && adopted.sequence != sequence { + adopted = nil + } + state.adopt = nil _, declaresSkip := def.Edges["skipped"] - skip := !isHuman && declaresSkip && autoSkipEligible(def.Configs, state) + skip := adopted == nil && !isHuman && declaresSkip && autoSkipEligible(def.Configs, state) state.nextSequence = sequence + 1 state.visits[key]++ // Publish the run's server-captured changed paths into the input memory @@ -1643,13 +1685,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na retryInput = input state.retryOverride = nil } - if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil { - if ctx.Err() != nil { - finish("canceled", "", ctx.Err().Error()) + if adopted == nil { + if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil { + if ctx.Err() != nil { + canceled() + return + } + finish("failed", "error", err.Error()) return } - finish("failed", "error", err.Error()) - return } // queued -> running: report the new current stage and the step start. r.emitRun(protocol.RunEventFrame{ @@ -1668,7 +1712,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil { r.mu.Unlock() if ctx.Err() != nil { - finish("canceled", "", ctx.Err().Error()) + canceled() return } finish("failed", "error", err.Error()) @@ -1744,9 +1788,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na var publicBefore map[string]bool if stage.EffectiveType(def.Type) == stage.ModelType { visitTurn = r.turnPaths(id, sequence, key) - if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil { - finish("failed", "error", turnErr.Error()) - return + if adopted == nil { + if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil { + finish("failed", "error", turnErr.Error()) + return + } } artifactsDir = visitTurn.TurnDir def.ArtifactsDir = artifactsDir @@ -1779,6 +1825,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na var piClient *stage.PiClient var err error stageCtx := ctx + if adopted != nil { + // Attach to the surviving session instead of prompting again. + client.Backend, client.Reference = adopted.owner.Backend, adopted.owner.Reference + stageCtx = stage.WithResumeSession(stageCtx, adopted.owner.ID) + } var modelConfig stage.ModelConfig var budgetBefore stage.BudgetSnapshot if stage.EffectiveType(def.Type) == stage.ModelType { @@ -1846,6 +1897,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na stageSession = client.SessionID } else if stage.EffectiveType(def.Type) == stage.ModelType { plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir} + if adopted != nil { + plain.Backend, plain.Reference = adopted.owner.Backend, adopted.owner.Reference + } plain.OnGenerationSession = func(session, backend, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1860,6 +1914,13 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na result, err = stage.DefaultRegistry().Run(stageCtx, def, input) } }() + if r.detach && stage.Detached(ctx) { + // The server is stopping. The stage left its session running and + // owned (or never admitted one); nothing about this visit is settled + // here. The next process re-adopts the session and continues. + log.Printf("run %d stage %s #%d: server stopping; stage left running for the next start to reattach", id, key, sequence) + return + } ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second) activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id) ownershipCancel() @@ -1877,6 +1938,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na after := filediff.Capture(snapshotCtx, workspace, before) snapshotCancel() changes := filediff.Compare(before, after) + if adopted != nil && changes.Status == "complete" { + // The pre-visit snapshot belonged to the stopped process; only the + // changes made after reattaching were compared. + changes.Status = "partial" + changes.Diagnostic = "Server restarted during this stage; changes made before the restart are not listed." + } if errors.Is(err, stage.ErrStageCleanup) { changes.Status = "partial" changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing." @@ -2062,7 +2129,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na return } if ctx.Err() != nil { - finish("canceled", "", ctx.Err().Error()) + canceled() return } // Advance the in-memory UI-surface evidence only after the step is @@ -2099,7 +2166,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na }, }, false) if ctx.Err() != nil { - finish("canceled", "", ctx.Err().Error()) + canceled() return } if terminal { diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 215dd369..58e1ca22 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -58,6 +58,12 @@ type Options struct { // target-aware verification. Definitions can name only a registered ID; they // cannot select an endpoint or supply provenance. VerificationFixtures []VerificationFixture + // DetachSessions keeps harness sessions running across a server restart: + // a stopping server leaves stage and chat sessions running (DetachSessions + // on the handler records the chats) and the next start reattaches them. + // False keeps the previous behaviour (tests, embedded servers): a stopping + // server interrupts its runs and the next start marks them interrupted. + DetachSessions bool } // Handler serves the persistent connection with default options. Canceling ctx @@ -77,7 +83,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin var runs *runner var runErr error if len(stores) > 0 { - runs, runErr = newRunnerAt(ctx, stores[0], options.ArtifactsRoot) + runs, runErr = newRunnerReattaching(ctx, stores[0], options.ArtifactsRoot, options.DetachSessions) if runErr == nil { runs.events = events } @@ -115,15 +121,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin settingsStore = stores[0] } hub.settings = settingsStore - if options.Chat != nil && registry != nil { - // Keep observing chats whose turn a previous process left running, so - // their reply or question still notifies with no client attached. - go hub.observeRunningTurns(ctx, options.Chat) - } if len(stores) > 0 { // Prompts queued behind a running turn are durable and owned by their // session; resume delivering any a previous process left queued. hub.queueStore = stores[0] + hub.liveStore = stores[0] if options.Chat != nil { go hub.resumeQueued(ctx, options.Chat) } @@ -449,12 +451,44 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } } mux.HandleFunc("GET /connect", connect) - return &runHandler{Handler: mux, runner: runs} + if options.Chat != nil && registry != nil { + // Reattach the chats a previous process left live (and fold in turns + // that ended while it was down), then keep observing any other chat + // whose turn is still running, so a reply or question still notifies + // with no client attached. + go func() { + hub.reattachSessions(ctx, options.Chat) + hub.observeRunningTurns(ctx, options.Chat) + }() + } + if runs != nil { + // Continue the runs a previous process left in flight, now that the + // runner's collaborators are wired and before any request is served. + runs.reattachRuns() + } + return &runHandler{Handler: mux, runner: runs, hub: hub} } type runHandler struct { http.Handler runner *runner + hub *chatHub +} + +// DetachSessions records the chats left live on their harness so the next +// start reattaches them. Call it when the server is stopping, before the +// handler's context is canceled (rooms are torn down after that) and before the +// store closes. It never interrupts a session. +func DetachSessions(handler http.Handler) { + h, ok := handler.(*runHandler) + if !ok || h.hub == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := h.hub.detachSessions(ctx); err != nil { + log.Printf("record live chat sessions: %v", err) + } } // WaitRuns drains active execution before the database is closed on shutdown. diff --git a/internal/stage/agent.go b/internal/stage/agent.go index b7a57c1c..e71b7e6f 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -27,6 +27,23 @@ type AgentClient struct { WorkStarted bool } +// resumeConnection reopens the retained execution service a previous server +// process recorded for a session, refusing a different backend or directory. +func resumeConnection(reference, backend, directory string) (*opencode.Connection, error) { + if reference == "" { + return nil, fmt.Errorf("retained OpenCode session has no execution-service reference") + } + connection, err := opencode.FromReference(reference) + if err != nil { + return nil, err + } + if (backend != "" && connection.Backend != backend) || connection.Directory != directory { + connection.Close() + return nil, fmt.Errorf("retained OpenCode execution context mismatch") + } + return connection, nil +} + func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error { return c.interrupt(ctx, id, false) } @@ -80,7 +97,18 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s if c.Operations != nil { c.Operations.Provider, c.Operations.Model = config.Provider, config.Model } - if c.command == nil { + // A re-adopted session was admitted by a previous server process: attach + // to its retained service and wait for it; never create or prompt again. + resume := TakeResumeSession(ctx) + if resume != "" && c.command == nil { + connection, err := resumeConnection(c.Reference, c.Backend, c.Directory) + if err != nil { + return "", err + } + defer connection.Close() + c.connection = connection + c.Backend = connection.Backend + } else if c.command == nil { connection, err := opencode.OpenStage(ctx, opencode.Executable(), c.Directory, c.Backend) if err != nil { // Connecting to the service failed before any session existed. @@ -101,24 +129,31 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s if err != nil { return "", err } - data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules}) - if err != nil { - if variant := model["variant"]; variant != "" { - err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err) + if resume != "" { + if c.Backend != "v2" || !sessionIDPattern.MatchString(resume) { + return "", fmt.Errorf("retained OpenCode session %.128s cannot be re-adopted on backend %q", RedactEvidence(resume), c.Backend) } - // No prompt or tool call was admitted: a request failure here is a - // reachable retry rather than a reason to end the run. - return "", harnessUnreachable(err) + c.SessionID = resume + } else { + data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules}) + if err != nil { + if variant := model["variant"]; variant != "" { + err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err) + } + // No prompt or tool call was admitted: a request failure here is a + // reachable retry rather than a reason to end the run. + return "", harnessUnreachable(err) + } + var created struct { + Data struct { + ID string `json:"id"` + } `json:"data"` + } + if json.Unmarshal(data, &created) != nil || !sessionIDPattern.MatchString(created.Data.ID) { + return "", fmt.Errorf("invalid agent session") + } + c.SessionID = created.Data.ID } - var created struct { - Data struct { - ID string `json:"id"` - } `json:"data"` - } - if json.Unmarshal(data, &created) != nil || !sessionIDPattern.MatchString(created.Data.ID) { - return "", fmt.Errorf("invalid agent session") - } - c.SessionID = created.Data.ID if c.connection != nil && c.Backend == "v1" { c.Reference, err = c.connection.Reference() if err != nil { @@ -134,6 +169,12 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s defer func() { watch.stopWatch(ctx, &err) defer watch.finish(ctx, &err) + if err != nil && Detached(ctx) { + // The server is restarting: leave the admitted session running and + // its ownership recorded, so the next process re-adopts it. + err = fmt.Errorf("%w: retained OpenCode session %s left running", ErrDetached, c.SessionID) + return + } cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) defer cancel() if err != nil { @@ -181,8 +222,10 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s return "", err } c.WorkStarted = true - if _, err = c.call(ctx, "post", "/api/session/"+c.SessionID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil { - return "", err + if resume == "" { + if _, err = c.call(ctx, "post", "/api/session/"+c.SessionID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil { + return "", err + } } if c.Backend == "v1" { // Native V1 prompts block until completion; keep the existing settlement @@ -193,6 +236,7 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s } else if err = c.waitForSessionIdle(ctx, c.SessionID); err != nil { return "", err } + var data []byte data, text, err = c.readFinalAgentResult(ctx, c.SessionID) if len(data) > 0 { c.Transcript = string(data) diff --git a/internal/stage/detach.go b/internal/stage/detach.go new file mode 100644 index 00000000..0bf86d7e --- /dev/null +++ b/internal/stage/detach.go @@ -0,0 +1,61 @@ +package stage + +import ( + "context" + "errors" + "sync" +) + +// A Pipelines server restart must not stop harness work. The runner cancels a +// run's context with ErrServerShutdown as its cause when the server process is +// stopping (as opposed to an operator cancel, whose cause is context.Canceled). +// A stage client that sees this cause leaves its admitted session running: it +// neither interrupts it nor clears its recorded ownership, and returns an error +// wrapping ErrDetached. The next server process re-adopts the session through +// WithResumeSession and waits for it instead of prompting again. + +// ErrServerShutdown is the cancellation cause of a run context whose server is +// stopping or restarting. +var ErrServerShutdown = errors.New("pipelines server is shutting down; harness sessions are left running") + +// ErrDetached reports that a stage left its admitted harness session running +// for the next server process to re-adopt. +var ErrDetached = errors.New("stage session detached for server restart") + +// Detached reports whether ctx was canceled because the server is stopping. +func Detached(ctx context.Context) bool { + return ctx.Err() != nil && errors.Is(context.Cause(ctx), ErrServerShutdown) +} + +type resumeKey struct{} + +type resumeHolder struct { + mu sync.Mutex + session string +} + +// WithResumeSession marks the next model generation under ctx as a re-adoption +// of an already admitted session: the client attaches to sessionID, waits for it +// to settle and reads its final result, without creating a session or sending a +// prompt. The mark is consumed by the first generation, so a later retry or +// repair attempt starts fresh as usual. +func WithResumeSession(ctx context.Context, sessionID string) context.Context { + if sessionID == "" { + return ctx + } + return context.WithValue(ctx, resumeKey{}, &resumeHolder{session: sessionID}) +} + +// TakeResumeSession returns and consumes the session marked by +// WithResumeSession, or "" when the generation is a fresh one. +func TakeResumeSession(ctx context.Context) string { + holder, _ := ctx.Value(resumeKey{}).(*resumeHolder) + if holder == nil { + return "" + } + holder.mu.Lock() + defer holder.mu.Unlock() + id := holder.session + holder.session = "" + return id +} diff --git a/internal/stage/detach_test.go b/internal/stage/detach_test.go new file mode 100644 index 00000000..442a7f5e --- /dev/null +++ b/internal/stage/detach_test.go @@ -0,0 +1,159 @@ +package stage + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +// fakeV2Session answers the V2 calls a stage makes for one session, records +// every request, and reports the session running until finish is closed. +type fakeV2Session struct { + t *testing.T + id string + finish chan struct{} + mu sync.Mutex + calls []string +} + +func (f *fakeV2Session) command(_ context.Context, _ string, args ...string) ([]byte, error) { + f.mu.Lock() + f.calls = append(f.calls, args[1]+" "+args[2]) + f.mu.Unlock() + switch path := args[2]; { + case path == "/api/session": + return []byte(`{"data":{"id":"` + f.id + `"}}`), nil + case strings.HasSuffix(path, "/prompt"), strings.HasSuffix(path, "/interrupt"): + return []byte(`{}`), nil + case path == "/api/session/active": + select { + case <-f.finish: + return []byte(`{"data":{}}`), nil + default: + return []byte(`{"data":{"` + f.id + `":{"type":"running"}}}`), nil + } + case path == "/api/session/"+f.id+finalAssistantReadback: + return []byte(`{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n\nreattached reply\n"}]}]}`), nil + } + return nil, fmt.Errorf("unexpected request %v", args) +} + +func (f *fakeV2Session) called(prefix string) bool { + f.mu.Lock() + defer f.mu.Unlock() + for _, call := range f.calls { + if strings.HasPrefix(call, prefix) { + return true + } + } + return false +} + +// TestAgentDetachesOnServerShutdown: a server shutdown (not an operator cancel) +// leaves the admitted session running and owned: no interrupt, no release. +func TestAgentDetachesOnServerShutdown(t *testing.T) { + fake := &fakeV2Session{t: t, id: "ses_detach", finish: make(chan struct{})} + var released []string + client := &AgentClient{OnSession: func(id string, active bool) error { + if !active { + released = append(released, id) + } + return nil + }} + client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond + ctx, cancel := context.WithCancelCause(context.Background()) + go func() { + for !fake.called("post /api/session/ses_detach/prompt") { + time.Sleep(time.Millisecond) + } + cancel(ErrServerShutdown) + }() + _, err := client.Generate(ctx, ModelConfig{}, "work") + if !errors.Is(err, ErrDetached) { + t.Fatalf("shutdown did not detach: %v", err) + } + if fake.called("post /api/session/ses_detach/interrupt") { + t.Fatal("a server shutdown interrupted the stage session") + } + if len(released) != 0 { + t.Fatalf("a detached session's ownership was released: %v", released) + } +} + +// TestAgentOperatorCancelStillInterrupts keeps the explicit stop path. +func TestAgentOperatorCancelStillInterrupts(t *testing.T) { + fake := &fakeV2Session{t: t, id: "ses_cancel", finish: make(chan struct{})} + client := &AgentClient{} + client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + go func() { + for !fake.called("post /api/session/ses_cancel/prompt") { + time.Sleep(time.Millisecond) + } + cancel() + // Interrupt confirmation waits for the session to settle. + for !fake.called("post /api/session/ses_cancel/interrupt") { + time.Sleep(time.Millisecond) + } + close(fake.finish) + }() + if _, err := client.Generate(ctx, ModelConfig{}, "work"); err == nil || errors.Is(err, ErrDetached) { + t.Fatalf("operator cancel did not fail the stage: %v", err) + } + if !fake.called("post /api/session/ses_cancel/interrupt") { + t.Fatal("operator cancel did not interrupt the session") + } +} + +// TestAgentResumeAttachesWithoutPrompting: a re-adopted session is waited on +// and read back; it is never created or prompted again, and the resume mark is +// consumed so a retry starts fresh. +func TestAgentResumeAttachesWithoutPrompting(t *testing.T) { + fake := &fakeV2Session{t: t, id: "ses_adopted", finish: make(chan struct{})} + var owned []bool + client := &AgentClient{OnSession: func(id string, active bool) error { + if id != "ses_adopted" { + t.Errorf("ownership for %s", id) + } + owned = append(owned, active) + return nil + }} + client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond + ctx := WithResumeSession(context.Background(), "ses_adopted") + go func() { + time.Sleep(20 * time.Millisecond) + close(fake.finish) + }() + text, err := client.Generate(ctx, ModelConfig{}, "work") + if err != nil || !strings.Contains(text, "reattached reply") { + t.Fatalf("resume = %q, %v", text, err) + } + if fake.called("post /api/session ") || fake.called("post /api/session/ses_adopted/prompt") { + t.Fatalf("a re-adopted session was created or prompted again: %v", fake.calls) + } + if len(owned) != 2 || !owned[0] || owned[1] { + t.Fatalf("ownership was not re-asserted then released: %v", owned) + } + if TakeResumeSession(ctx) != "" { + t.Fatal("the resume mark was not consumed") + } +} + +// TestPlainResumeReadsResultReachedWhileDown: a text-only session that ended +// during the downtime is folded in from its final assistant message. +func TestPlainResumeReadsResultReachedWhileDown(t *testing.T) { + fake := &fakeV2Session{t: t, id: "ses_plain", finish: make(chan struct{})} + close(fake.finish) + client := &OpenCodeClient{Backend: "v2", Directory: "/work", command: fake.command, sessionPollInterval: time.Millisecond} + text, err := client.Generate(WithResumeSession(context.Background(), "ses_plain"), ModelConfig{}, "prompt") + if err != nil || !strings.Contains(text, "reattached reply") { + t.Fatalf("resume = %q, %v", text, err) + } + if fake.called("post /api/session/ses_plain/prompt") || fake.called("post /api/session ") { + t.Fatalf("a re-adopted session was prompted again: %v", fake.calls) + } +} diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index d65b39ac..f47643e2 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -273,7 +273,18 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp if c.Operations != nil { c.Operations.Provider, c.Operations.Model = config.Provider, config.Model } - if c.command == nil { + // A re-adopted session was admitted by a previous server process (see + // WithResumeSession): wait for it and read its result, never prompt again. + resume := TakeResumeSession(ctx) + if resume != "" && c.command == nil { + connection, err := resumeConnection(c.Reference, c.Backend, c.Directory) + if err != nil { + return "", err + } + defer connection.Close() + c.connection = connection + c.Backend = connection.Backend + } else if c.command == nil { connection, err := opencode.OpenStage(ctx, opencode.Executable(), c.Directory, c.Backend) if err != nil { // Connecting to the service failed before any session existed. @@ -313,33 +324,45 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp if err != nil { return "", err } - created, err := c.call(ctx, "post", "/api/session", map[string]any{ - "title": "Pipeline stage generation", - "location": map[string]string{"directory": directory}, - "model": model, - "permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}}, - }) - if err != nil { - if variant := model["variant"]; variant != "" { - err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err) - } - // No prompt or tool call was admitted: a request failure here is a - // reachable retry rather than a reason to end the run. - return "", harnessUnreachable(err) - } var session struct { Data struct { ID string `json:"id"` } `json:"data"` } - if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) { - return "", fmt.Errorf("OpenCode returned an invalid session ID") + if resume != "" { + if c.Backend != "v2" || !sessionIDPattern.MatchString(resume) { + return "", fmt.Errorf("retained OpenCode session %.128s cannot be re-adopted on backend %q", RedactEvidence(resume), c.Backend) + } + session.Data.ID = resume + } else { + created, err := c.call(ctx, "post", "/api/session", map[string]any{ + "title": "Pipeline stage generation", + "location": map[string]string{"directory": directory}, + "model": model, + "permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}}, + }) + if err != nil { + if variant := model["variant"]; variant != "" { + err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err) + } + // No prompt or tool call was admitted: a request failure here is a + // reachable retry rather than a reason to end the run. + return "", harnessUnreachable(err) + } + if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) { + return "", fmt.Errorf("OpenCode returned an invalid session ID") + } } var watch *budgetWatch promptStarted := false defer func() { watch.stopWatch(ctx, &err) defer watch.finish(ctx, &err) + if err != nil && Detached(ctx) { + // The server is restarting: leave the session running and owned. + err = fmt.Errorf("%w: retained OpenCode session %s left running", ErrDetached, session.Data.ID) + return + } cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() if err != nil { @@ -392,8 +415,10 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp // including unbudgeted text-only stages, so its exact native turns survive. if c.Backend == "v2" { promptStarted = true - if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil { - return "", err + if resume == "" { + if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil { + return "", err + } } if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil { return "", err