Keep OpenCode/Codex chat and stage sessions running across server restarts

A stopping server no longer interrupts harness sessions. Runs are canceled
with a shutdown cause, so stage clients leave admitted sessions running and
owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait,
read back the result, continue the graph), restarts unadmitted visits,
continues runs between stages, and falls back to interrupted with a reason
when the harness is gone. Chats with live work are recorded on shutdown and
reattached on start; turns that ended during the downtime are folded in
from history. The server-spawned V2 chat serve is now a detached owned
service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
This commit is contained in:
2026-11-04 16:17:08 +00:00
parent 9351cd7d74
commit aa8eddd686
26 changed files with 2318 additions and 99 deletions
+61 -5
View File
@@ -49,8 +49,18 @@ func main() {
if err := store.AcquireServerLock(); err != nil {
log.Fatal(err)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
signals, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
// The server's own context ends after a signal, once the chats left live on
// their harness have been recorded (see the shutdown goroutine below).
ctx, cancelServer := context.WithCancel(context.Background())
defer cancelServer()
// A restart or stop leaves harness sessions running and the next start
// reattaches them; PIPELINES_SHUTDOWN_SESSIONS=stop restores stopping them.
detach, err := shutdownDetaches(os.Getenv(shutdownSessionsEnvKey))
if err != nil {
log.Fatal(err)
}
// A brand-new database starts with the bundled examples; an existing or
// already-seeded one is never changed, even if its pipelines were deleted.
if seeded, err := server.SeedExamples(ctx, store); err != nil {
@@ -62,7 +72,7 @@ func main() {
// Each new run gets a bounded one-line title from the configured default
// harness; the summariser falls back to the prompt's first line, so title
// generation never blocks or fails run creation.
options := server.Options{AllowAuthoring: *allowAuthoring, ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)}
options := server.Options{AllowAuthoring: *allowAuthoring, ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv), DetachSessions: detach}
recorded := ""
if value, err := store.OpenCodeEndpoint(ctx); err != nil {
log.Printf("OpenCode endpoint setting unreadable: %v", err)
@@ -117,7 +127,7 @@ func main() {
}
}
if options.Chat == nil {
if conn, err := opencode.OpenChat(ctx, opencode.Executable(), ""); err != nil {
if conn, err := openChat(ctx, detach); err != nil {
log.Printf("OpenCode chat unavailable: %v", err)
} else if conn.URL == "" {
conn.Close()
@@ -141,7 +151,11 @@ func main() {
chats := server.NewChatManager(ctx, options.Chat, store)
if len(chats.Backends()) > 0 {
options.Chat = chats
defer chats.Close()
if detach {
defer chats.DetachSessions()
} else {
defer chats.Close()
}
}
srv := &http.Server{Addr: *addr, Handler: server.HandlerWithOptions(ctx, options, store), ReadHeaderTimeout: 5 * time.Second}
// Deferred before WaitRuns so it runs after the runs settle: the owned stage
@@ -149,7 +163,16 @@ func main() {
defer opencode.StopIdleStageService()
defer server.WaitRuns(srv.Handler)
go func() {
<-ctx.Done()
select {
case <-signals.Done():
case <-ctx.Done():
}
if detach {
// Record the live chats while their rooms still exist.
server.DetachSessions(srv.Handler)
log.Printf("shutdown: harness sessions are left running; the next start reattaches them")
}
cancelServer()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
@@ -172,3 +195,36 @@ func main() {
log.Fatal(err)
}
}
// shutdownSessionsEnvKey selects what a stopping server does with harness
// sessions: "keep" (the default) leaves them running for the next start to
// reattach; "stop" interrupts runs and closes chat clients as before.
const shutdownSessionsEnvKey = "PIPELINES_SHUTDOWN_SESSIONS"
func shutdownDetaches(value string) (bool, error) {
switch value {
case "", "keep":
return true, nil
case "stop":
return false, nil
}
return false, errors.New(shutdownSessionsEnvKey + " must be keep or stop")
}
// openChat opens the OpenCode service chat brokers from when no endpoint is
// attached. When sessions are kept across restarts, a V2 installation uses the
// detached Pipelines-owned chat service, which outlives this process; otherwise
// (and for V1) it is a serve child that ends with the server.
func openChat(ctx context.Context, detach bool) (*opencode.Connection, error) {
executable := opencode.Executable()
if detach {
if backend, err := opencode.Detect(ctx, executable); err == nil && backend == "v2" {
directory, err := os.Getwd()
if err != nil {
return nil, err
}
return opencode.EnsureOwnedChatService(ctx, executable, directory)
}
}
return opencode.OpenChat(ctx, executable, "")
}
+39
View File
@@ -6,6 +6,45 @@ when the `claude` CLI is installed. The selected backend is stored on the
operator-session row and returned as `backend` on every `chat_sessions` row, so a
restart does not have to guess from the session id.
## Restarts keep sessions running
A server restart or stop does not stop chat sessions. Only an operator action
stops one: closing the chat, or Stop on its turn. On shutdown the server records
each chat that still has work on its harness (a turn in flight, a pending
question or approval, or running background tasks) in `chat_live_sessions`.
It then detaches each backend instead of closing it, and never sends an
interrupt.
On the next start each recorded chat is reattached. The server asks the harness
whether the turn is still running, re-subscribes to its event stream with no
client attached, re-reads pending approvals/questions and background shells,
and holds queued prompts behind the turn. If a turn ended while the server was
down, or its end event was lost with the old stream, the end is folded in from
the harness's history. Clients get a `resync`, the reply notifies once (with
the text the harness recorded), and queued prompts are delivered in order. The
older start-up probe of recently used chats still runs for anything not
recorded, for example after a crash.
Per backend (the contract is `chatSessionDetacher` in
`internal/server/chat_detach.go`):
- **OpenCode**: sessions live in the OpenCode service. An attached endpoint
(the operator's CLI-managed service) already outlives the server. Without
one, a V2 server now uses a detached Pipelines-owned chat service
(`opencode serve` in its own session, registered in
`<cache>/pipelines/private/opencode-chat-service.json`). The next server
reuses it at the same address and password, and it is not stopped on
shutdown. A V1 serve child still ends with the server.
- **Codex**: threads and turns live in the App Server. Detaching closes the
control connection, and the next server reconciles through `thread/read`.
An approval or question the App Server sent to the old connection is bound to
it, so one still pending at shutdown cannot be answered after the restart.
- **Claude Code**: closed as before until its host implements
`DetachSessions`.
`PIPELINES_SHUTDOWN_SESSIONS=stop` restores the old shutdown, which closes the
chat clients and records nothing. See [Installation](install.md#restarts-keep-sessions-running).
## Codex
The adapter in `internal/codex` connects to the Codex App Server's Unix
+4 -1
View File
@@ -92,7 +92,10 @@ How the stage service behaves:
session is active on it. A busy service keeps its old limits until it is idle.
- **Lifetime.** It runs detached in its own process session, so a stopping
Pipelines service does not take active stage work with it. On server shutdown
it is stopped only when no session is active on it. Otherwise restart
it is stopped only when no session is active on it. A busy service keeps
running, and the next start re-adopts its sessions and continues their runs
(see [Installation](install.md#restarts-keep-sessions-running)). With
`PIPELINES_SHUTDOWN_SESSIONS=stop`, or when the service is gone, restart
recovery interrupts the retained sessions through it, as before.
- `pipelines debug` is unchanged: it opens a fresh interactive harness
conversation, not the stage session.
+46 -2
View File
@@ -173,8 +173,8 @@ another installation.
| Command | Effect |
| --- | --- |
| `pipelines start` | start server, web proxy and ntfy; print the startup report |
| `pipelines stop` | stop all three (active runs are interrupted; edits are not rolled back) |
| `pipelines restart` | stop then start, keeping data |
| `pipelines stop` | stop all three; harness sessions keep running and the next start reattaches them (see below) |
| `pipelines restart` | stop then start, keeping data; chat turns and stage sessions continue |
| `pipelines doctor` | startup report, health of server and ntfy, OpenCode API check (no inference) |
| `pipelines status` | server, host and ntfy publishing health, from the running server |
| `pipelines logs` | follow `logs/server.log` |
@@ -184,6 +184,50 @@ another installation.
| `pipelines service disable` | disable autostart and stop the supervisor |
| `pipelines service status` | autostart and supervisor state |
### Restarts keep sessions running
A server restart, `pipelines stop` and `pipelines update` stop the Pipelines
processes, not the work running on harnesses. On SIGTERM the server leaves
every OpenCode and Codex session running, records what it needs to reattach,
and exits. It does not interrupt or abort anything. On the next start:
- **Runs**: a stage whose OpenCode V2 session is still alive is re-adopted. The
runner waits for that same session (no new prompt), or reads the result it
reached while the server was down, and then continues the graph. A stage
that had not yet handed work to a harness starts again. A run between stages
continues with its next stage. The stage timeout restarts when the stage is
reattached. Its file-change list is marked partial, because the snapshot
taken before the restart is gone.
- **Chats**: turns, pending questions/approvals, background tasks and queued
prompts are picked up again, and a reply that arrived during the downtime
still notifies. See [Chat backends](chat-backends.md#restarts-keep-sessions-running).
If the harness is gone, the old behaviour applies and the run is interrupted
with the reason in its diagnostic, for example "Server restarted; the OpenCode
service running stage … was gone". This happens when the OpenCode service
process died, or the session belongs to a Pi or OpenCode V1 process that ended
with the server.
Only explicit operator actions stop sessions:
- `pipelines cancel <run>` (or Cancel in the app) interrupts a run's stage.
- Closing a chat, or Stop on its turn, interrupts that chat.
To stop everything: cancel the active runs and close or stop the busy chats,
then run `pipelines stop`. The owned OpenCode stage service stops itself at
shutdown when idle. The owned chat service, used only without an attached
endpoint, keeps running and is reused. Stop it by killing the `pid` in
`~/Library/Caches/pipelines/private/opencode-chat-service.json`
(`~/.cache/...` on Linux). To make every shutdown interrupt runs and close chat
clients as before, add `"PIPELINES_SHUTDOWN_SESSIONS": "stop"` to
`opencode_env` in `<prefix>/pipelines-install.json` and restart.
Limit on Linux: systemd stops every process in the service's cgroup. The
OpenCode services the server starts (stage service, owned chat service) end
with `pipelines stop`/`restart`, and their runs fall back as described above.
An attached OpenCode endpoint and the Codex App Server are separate and
survive. macOS launchd does not kill them.
The login service is `~/Library/LaunchAgents/net.a-wan.pipelines.plist` on
macOS and `~/.config/systemd/user/net.a-wan.pipelines.service` on Linux. It
runs `manage.py supervise`, which owns all three processes. This is login
+8
View File
@@ -176,6 +176,14 @@ func (c *Chat) Close() {
_ = c.conn.Close(websocket.StatusNormalClosure, "")
}
// DetachSessions is the server-restart shutdown: threads and their turns live
// in the Codex App Server, not in this client, so closing the control
// connection leaves them running; no turn is interrupted. The next server
// process reconnects and reconciles each thread through thread/read. An
// approval or question the App Server sent to this connection is bound to it,
// so one still pending at shutdown is not answerable after the restart.
func (c *Chat) DetachSessions() { c.Close() }
func (c *Chat) readLoop() {
for {
_, data, err := c.conn.Read(c.ctx)
+48
View File
@@ -0,0 +1,48 @@
package codex
import (
"context"
"encoding/json"
"sync"
"sync/atomic"
"testing"
)
// A server restart detaches from the App Server without interrupting the
// thread's turn; the next server's fresh connection still sees the turn in
// progress, then completed, through the read-only turn list.
func TestDetachSessionsLeavesTurnRunningForNextConnection(t *testing.T) {
var completed atomic.Bool
var mu sync.Mutex
var methods []string
appServer := func(method string, _ json.RawMessage) any {
mu.Lock()
methods = append(methods, method)
mu.Unlock()
status := "inProgress"
if completed.Load() {
status = "completed"
}
return json.RawMessage(`{"data":[{"id":"turn","status":"` + status + `","items":[]}],"nextCursor":null}`)
}
before := modelTestChat(t, appServer)
if running, err := before.ProbeTurn(context.Background(), fixtureThread); err != nil || !running {
t.Fatalf("turn not running before the restart: %v %v", running, err)
}
before.DetachSessions()
after := modelTestChat(t, appServer)
if running, err := after.ProbeTurn(context.Background(), fixtureThread); err != nil || !running {
t.Fatalf("turn did not survive the restart: %v %v", running, err)
}
completed.Store(true) // finished while nobody was watching
if running, err := after.ProbeTurn(context.Background(), fixtureThread); err != nil || running {
t.Fatalf("completed turn still reported running: %v %v", running, err)
}
mu.Lock()
defer mu.Unlock()
for _, method := range methods {
if method == "turn/interrupt" {
t.Fatalf("detaching interrupted the turn: %v", methods)
}
}
}
+7
View File
@@ -304,6 +304,13 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error {
return nil
}
// DetachSessions is the server-restart shutdown. Sessions, turns, questions,
// permissions and shells live in the OpenCode service, not in this adapter, so
// there is nothing to stop: the connection's owner decides whether the service
// itself outlives the server (an attached endpoint and the detached owned chat
// service do; a V1 serve child does not).
func (c *Chat) DetachSessions() {}
// Messages reads one session's stored turns. On the owned serve backend it
// reuses Connection.Messages, including its read-only OutputFormat readback
// fallback, instead of duplicating that path. It maps the verified assistant
+64
View File
@@ -0,0 +1,64 @@
package opencode
import (
"context"
"errors"
"strings"
"syscall"
"testing"
)
// The server-spawned chat service is detached and registered: a second server
// process reuses the same running service (sessions survive the restart), and
// it is separate from the stage service.
func TestOwnedChatServiceIsReusedAcrossServerProcesses(t *testing.T) {
executable := fakeOwnedStageEnv(t)
t.Setenv(ChatConfigEnvKey, "")
t.Cleanup(func() {
path, _ := ChatServiceRegistrationPath()
if state, err := readOwnedStageState(path); err == nil {
_ = syscall.Kill(state.PID, syscall.SIGKILL)
}
})
first, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir())
if err != nil {
t.Fatal(err)
}
path, _ := ChatServiceRegistrationPath()
stagePath, _ := StageServiceRegistrationPath()
if first.credentialFile != path || path == stagePath || !strings.HasPrefix(first.URL, "http://127.0.0.1:") {
t.Fatalf("chat connection = %+v", first)
}
pid := ownedPID(t, first)
first.Close() // the previous server process exits
second, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir())
if err != nil || second.URL != first.URL || ownedPID(t, second) != pid {
t.Fatalf("chat service was not reused by the next server: %v", err)
}
}
// ReferenceAlive tells a live retained service from one whose process is gone,
// without reviving it.
func TestReferenceAliveDoesNotRevive(t *testing.T) {
executable := fakeOwnedStageEnv(t)
c, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
if err != nil {
t.Fatal(err)
}
reference, err := c.Reference()
if err != nil {
t.Fatal(err)
}
if err := ReferenceAlive(context.Background(), reference); err != nil {
t.Fatalf("live service reported gone: %v", err)
}
pid := ownedPID(t, c)
_ = syscall.Kill(pid, syscall.SIGKILL)
waitGone(t, pid)
if err := ReferenceAlive(context.Background(), reference); !errors.Is(err, ErrServiceNotRunning) {
t.Fatalf("gone service not reported: %v", err)
}
if _, healthy := c.ownedInfo(context.Background()); healthy {
t.Fatal("ReferenceAlive revived the service")
}
}
+50 -7
View File
@@ -8,6 +8,7 @@ import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -90,23 +91,65 @@ func (c *Connection) Reference() (string, error) {
return f.Name(), nil
}
func FromReference(path string) (*Connection, error) {
type connectionReference struct{ Backend, URL, Password, Directory, CredentialFile string }
func readReference(path string) (connectionReference, error) {
var reference connectionReference
info, err := os.Lstat(path)
if err != nil {
return nil, err
return reference, err
}
if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || info.Size() > MaxRequest {
return nil, fmt.Errorf("invalid private OpenCode connection reference")
return reference, fmt.Errorf("invalid private OpenCode connection reference")
}
if stat, ok := info.Sys().(*syscall.Stat_t); !ok || stat.Uid != uint32(os.Getuid()) {
return nil, fmt.Errorf("OpenCode reference belongs to another user")
return reference, fmt.Errorf("OpenCode reference belongs to another user")
}
data, err := os.ReadFile(path)
if err != nil {
return nil, err
return reference, err
}
var reference struct{ Backend, URL, Password, Directory, CredentialFile string }
if err = json.Unmarshal(data, &reference); err != nil {
err = json.Unmarshal(data, &reference)
return reference, err
}
// ErrServiceNotRunning reports that a retained OpenCode service no longer
// answers: its process is gone, and with it any session turn it was running.
var ErrServiceNotRunning = errors.New("the OpenCode service is not running")
// ReferenceAlive reports whether the execution service a retained reference
// names is running right now, without starting or reviving it. Restart recovery
// uses it to tell a session that kept running across a Pipelines restart from
// one whose harness process is gone.
func ReferenceAlive(ctx context.Context, path string) error {
reference, err := readReference(path)
if err != nil {
return fmt.Errorf("retained OpenCode reference unreadable: %w", err)
}
if reference.Backend != "v2" {
return fmt.Errorf("the V1 OpenCode stage service belonged to the stopped server process")
}
if !filepath.IsAbs(reference.CredentialFile) {
return fmt.Errorf("invalid V2 service credential reference")
}
c, err := serviceConnection(reference.CredentialFile, reference.Directory)
if err != nil {
return err
}
if c.URL != reference.URL {
return fmt.Errorf("retained OpenCode service endpoint changed")
}
probe, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
if err := c.probeHealth(probe, "v2"); err != nil {
return fmt.Errorf("%w at %s", ErrServiceNotRunning, c.URL)
}
return nil
}
func FromReference(path string) (*Connection, error) {
reference, err := readReference(path)
if err != nil {
return nil, err
}
if reference.Backend == "v2" {
+60 -14
View File
@@ -54,14 +54,56 @@ type ownedStageState struct {
Identity string `json:"identity"`
}
// StageServiceRegistrationPath is the private registration of the owned stage
// service, beside the private connection references.
func StageServiceRegistrationPath() (string, error) {
// ownedService names one Pipelines-owned detached `opencode serve`: the stage
// service, or the chat service a server without an attached endpoint uses.
type ownedService struct {
name string // for logs: "stage" or "chat"
file string // registration file name, without .json
}
var (
stageOwnedService = ownedService{name: "stage", file: "opencode-stage-service"}
chatOwnedService = ownedService{name: "chat", file: "opencode-chat-service"}
)
func (s ownedService) registrationPath() (string, error) {
cache, err := os.UserCacheDir()
if err != nil {
return "", err
}
return filepath.Join(cache, "pipelines", "private", "opencode-stage-service.json"), nil
return filepath.Join(cache, "pipelines", "private", s.file+".json"), nil
}
// StageServiceRegistrationPath is the private registration of the owned stage
// service, beside the private connection references.
func StageServiceRegistrationPath() (string, error) {
return stageOwnedService.registrationPath()
}
// ChatServiceRegistrationPath is the private registration of the owned chat
// service (see EnsureOwnedChatService).
func ChatServiceRegistrationPath() (string, error) {
return chatOwnedService.registrationPath()
}
// EnsureOwnedChatService returns a connection to the detached Pipelines-owned
// V2 chat service, starting or reviving it when needed. Unlike a serve child of
// the server, it keeps running when the server stops, so chat turns, questions
// and background shells survive a restart; the next server reuses the same
// endpoint and password from the private registration. Chat-only limits
// (PIPELINES_OPENCODE_CHAT_CONFIG) apply to it alone.
func EnsureOwnedChatService(ctx context.Context, executable, directory string) (*Connection, error) {
overrides, err := chatConfigOverrides()
if err != nil {
return nil, err
}
content := ""
if overrides != nil {
if content, err = configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), overrides); err != nil {
return nil, err
}
}
return ensureOwnedService(ctx, chatOwnedService, content, executable, directory)
}
func isOwnedStageRegistration(path string) bool {
@@ -81,7 +123,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
if err != nil {
return nil, err
}
path, err := StageServiceRegistrationPath()
return ensureOwnedService(ctx, stageOwnedService, content, executable, directory)
}
func ensureOwnedService(ctx context.Context, service ownedService, content, executable, directory string) (*Connection, error) {
path, err := service.registrationPath()
if err != nil {
return nil, err
}
@@ -92,7 +138,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
defer ownedStageMu.Unlock()
unlock, err := lockFile(path + ".lock")
if err != nil {
return nil, fmt.Errorf("OpenCode stage service lock: %w", err)
return nil, fmt.Errorf("OpenCode %s service lock: %w", service.name, err)
}
defer unlock()
@@ -106,11 +152,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
case healthy && state.Identity == identity:
return ownedStageConnection(path, executable, directory)
case healthy && current.ownedIdle(ctx):
log.Printf("OpenCode stage service configuration changed; restarting the idle service")
log.Printf("OpenCode %s service configuration changed; restarting the idle service", service.name)
stopOwnedPID(pid)
case healthy:
// Never interrupt active stage work to apply a new limit.
log.Printf("OpenCode stage service configuration changed; keeping the busy service until it is idle")
log.Printf("OpenCode %s service configuration changed; keeping the busy service until it is idle", service.name)
return ownedStageConnection(path, executable, directory)
}
if u, err := url.Parse(state.URL); err == nil {
@@ -135,7 +181,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
case <-time.After(time.Duration(attempt) * time.Second):
}
}
started, err := startOwnedStage(ctx, executable, port, state.Password, content)
started, err := startOwnedStage(ctx, service, executable, port, state.Password, content)
if err == nil {
started.Identity = identity
if err := writeOwnedStageState(path, started); err != nil {
@@ -197,7 +243,7 @@ func ownedStageIdentity(executable, content string) string {
// startOwnedStage starts one detached serve process and waits until it answers
// the authenticated V2 info contract.
func startOwnedStage(ctx context.Context, executable string, port int, password, content string) (ownedStageState, error) {
func startOwnedStage(ctx context.Context, service ownedService, executable string, port int, password, content string) (ownedStageState, error) {
if port == 0 {
free, err := freePort()
if err != nil {
@@ -214,7 +260,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
// Its own session: a stopping Pipelines service (and its process group)
// does not take active stage work with it.
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
logPath, _ := StageServiceRegistrationPath()
logPath, _ := service.registrationPath()
logPath = strings.TrimSuffix(logPath, ".json") + ".log"
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
@@ -224,7 +270,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
cmd.Stdin = nil
if err := cmd.Start(); err != nil {
logFile.Close()
return ownedStageState{}, fmt.Errorf("OpenCode stage service could not start: %w", err)
return ownedStageState{}, fmt.Errorf("OpenCode %s service could not start: %w", service.name, err)
}
logFile.Close()
exited := make(chan struct{})
@@ -241,13 +287,13 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
}
select {
case <-exited:
return ownedStageState{}, fmt.Errorf("OpenCode stage service exited before becoming ready (see %s)", logPath)
return ownedStageState{}, fmt.Errorf("OpenCode %s service exited before becoming ready (see %s)", service.name, logPath)
case <-startup.Done():
_ = cmd.Process.Kill()
if ctx.Err() != nil {
return ownedStageState{}, ctx.Err()
}
return ownedStageState{}, fmt.Errorf("OpenCode stage service did not become ready within %s (see %s)", ownedStageStartTimeout, logPath)
return ownedStageState{}, fmt.Errorf("OpenCode %s service did not become ready within %s (see %s)", service.name, ownedStageStartTimeout, logPath)
case <-time.After(150 * time.Millisecond):
}
}
+90
View File
@@ -0,0 +1,90 @@
package pipeline
import (
"context"
"time"
)
// ChatLiveSession is one chat a stopping server left live on its harness: a
// turn in flight, an unanswered question or approval, or running background
// tasks. The next server process reattaches each one (and reads back the final
// state of a turn that ended while it was down).
type ChatLiveSession struct {
SessionID string
Backend string
// Turn is true when a turn was in flight; NativeTurnID names it when the
// harness reported one, so a reply is announced once.
Turn bool
NativeTurnID string
// Attention is true when a question/approval was pending or background
// tasks were running.
Attention bool
RecordedAt time.Time
}
func (s *Store) initChatLiveSessions() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS chat_live_sessions (
session_id TEXT PRIMARY KEY,
backend TEXT NOT NULL DEFAULT '',
turn INTEGER NOT NULL DEFAULT 0,
native_turn_id TEXT NOT NULL DEFAULT '',
attention INTEGER NOT NULL DEFAULT 0,
recorded_at INTEGER NOT NULL)`)
return err
}
// RecordChatLiveSessions replaces the recorded set with sessions.
func (s *Store) RecordChatLiveSessions(ctx context.Context, sessions []ChatLiveSession) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions`); err != nil {
return err
}
for _, session := range sessions {
at := session.RecordedAt
if at.IsZero() {
at = time.Now()
}
if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO chat_live_sessions(session_id,backend,turn,native_turn_id,attention,recorded_at) VALUES(?,?,?,?,?,?)`,
session.SessionID, session.Backend, session.Turn, session.NativeTurnID, session.Attention, at.UnixMilli()); err != nil {
return err
}
}
return tx.Commit()
}
// TakeChatLiveSessions returns the recorded set and clears it, so each start
// reattaches what the previous process recorded exactly once.
func (s *Store) TakeChatLiveSessions(ctx context.Context) ([]ChatLiveSession, error) {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return nil, err
}
defer tx.Rollback()
rows, err := tx.QueryContext(ctx, `SELECT session_id,backend,turn,native_turn_id,attention,recorded_at FROM chat_live_sessions ORDER BY recorded_at, session_id`)
if err != nil {
return nil, err
}
var sessions []ChatLiveSession
for rows.Next() {
var session ChatLiveSession
var at int64
if err := rows.Scan(&session.SessionID, &session.Backend, &session.Turn, &session.NativeTurnID, &session.Attention, &at); err != nil {
rows.Close()
return nil, err
}
session.RecordedAt = time.UnixMilli(at)
sessions = append(sessions, session)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `DELETE FROM chat_live_sessions`); err != nil {
return nil, err
}
return sessions, tx.Commit()
}
+41 -2
View File
@@ -190,18 +190,57 @@ func (s *Store) addColumnIfMissing(table, column, definition string) error {
return err
}
// RestartStep marks a visit that a stopped server had begun but had not yet
// handed to any harness session as interrupted, so its run can begin the same
// stage again under a new sequence. Nothing was admitted, so nothing replays.
func (s *Store) RestartStep(ctx context.Context, id int64, sequence int, diagnostic string) error {
return affected(s.db.ExecContext(ctx, "UPDATE run_steps SET status='interrupted',agent_active=0,diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE run_id=? AND sequence=? AND status='running' AND session_id=''", diagnostic, id, sequence))
}
func (s *Store) RecoverRuns(ctx context.Context) error {
return s.RecoverRunsKeeping(ctx, nil, nil)
}
// RecoverRunsKeeping is RecoverRuns for a server that reattaches work across a
// restart: runs in keep are left exactly as the previous process left them (the
// runner continues them), and a run in reasons is interrupted with that reason
// instead of the generic one, so the operator sees why it was not reattached.
func (s *Store) RecoverRunsKeeping(ctx context.Context, keep map[int64]bool, reasons map[int64]string) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
kept, keptArgs := "", []any{}
for id := range keep {
if kept != "" {
kept += ","
}
kept += "?"
keptArgs = append(keptArgs, id)
}
stepFilter, runFilter := "", ""
if kept != "" {
stepFilter = " AND run_id NOT IN (" + kept + ")"
runFilter = " AND id NOT IN (" + kept + ")"
}
// No replay: pending visits never began. A cleanup agent that was still
// active is marked interrupted, and its run is interrupted below.
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)", protocol.StepStatusWaiting); err != nil {
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET status=CASE WHEN (SELECT cancel_requested FROM pipeline_runs WHERE id=run_steps.run_id)=1 THEN 'canceled' ELSE 'interrupted' END,agent_active=0,diagnostic='Server stopped; stage was not resumed',finished_at=CURRENT_TIMESTAMP WHERE (status='running' OR agent_active=1) AND status<>? AND run_id IN (SELECT id FROM pipeline_runs WHERE cleanup_unconfirmed=0)"+stepFilter, append([]any{protocol.StepStatusWaiting}, keptArgs...)...); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
for id, reason := range reasons {
if keep[id] {
continue
}
if _, err := tx.ExecContext(ctx, "UPDATE run_steps SET diagnostic=? WHERE run_id=? AND status='interrupted' AND sequence=(SELECT MAX(sequence) FROM run_steps WHERE run_id=?)", "Server restarted; "+reason, id, id); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='interrupted',diagnostic=?,finished_at=CURRENT_TIMESTAMP WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 AND cleanup_unconfirmed=0", "Server restarted; "+reason+"; run was not resumed", id); err != nil {
return err
}
}
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0"+runFilter, append([]any{protocol.RunStatusWaiting}, keptArgs...)...); err != nil {
return err
}
// An export checkpoint is a paused handoff, not an interrupted model call.
+4
View File
@@ -93,6 +93,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
db.Close()
return nil, err
}
if err := s.initChatLiveSessions(); err != nil {
db.Close()
return nil, err
}
if err := s.initAppNotifications(); err != nil {
db.Close()
return nil, err
+243
View File
@@ -0,0 +1,243 @@
package server
import (
"context"
"log"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Chat sessions survive a Pipelines restart. Only an operator action (closing
// the chat, or Stop on its turn) stops one.
//
// Per-backend contract. A chat backend whose sessions can outlive the server
// process implements chatSessionDetacher. On shutdown the server calls
// DetachSessions instead of Close: the backend releases its own connection or
// process handles but must not interrupt, abort or kill any session, turn,
// question or background task. On the next start the server reattaches through
// the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still
// running?), Stream (re-subscribe), Permissions/Forms (pending questions and
// approvals), Shells (background tasks) and Messages (read back a turn that
// ended while the server was down). A backend without DetachSessions is closed
// as before, and its sessions end with the server.
//
// OpenCode (an attached or detached owned service) and Codex (its App Server)
// implement it; a Claude Code host that keeps its CLI processes alive can
// implement the same method.
type chatSessionDetacher interface {
DetachSessions()
}
// chatLiveStore records, on shutdown, the chats left live on their harness so
// the next start reattaches exactly those; *pipeline.Store implements it.
type chatLiveStore interface {
RecordChatLiveSessions(ctx context.Context, sessions []pipeline.ChatLiveSession) error
TakeChatLiveSessions(ctx context.Context) ([]pipeline.ChatLiveSession, error)
}
// chatReattachPoll is how often a reattached turn re-asks its harness whether
// it is still running, to fold in an end the re-subscribed stream did not see.
var chatReattachPoll = 2 * time.Second
// detachSessions records every chat with work still live on its harness: a
// turn in flight, a pending question or approval, or running background tasks.
// It must run before rooms are torn down (before the server context is
// canceled), and it never touches the sessions themselves.
func (h *chatHub) detachSessions(ctx context.Context) error {
if h.liveStore == nil {
return nil
}
h.mu.Lock()
var live []pipeline.ChatLiveSession
for id, room := range h.rooms {
turn := room.running || room.observedTurn
if step := h.turns[id]; step != nil && step.step != nil && step.step.Status == protocol.ChatStepStarted && !step.step.Intermediate {
turn = true
}
attention := len(room.pending) > 0 || len(room.forms) > 0 || h.backgroundTasksLocked(id) > 0
if turn || attention {
live = append(live, pipeline.ChatLiveSession{SessionID: id, Turn: turn, NativeTurnID: room.nativeTurnID, Attention: attention, RecordedAt: h.clock()})
}
}
h.mu.Unlock()
if len(live) > 0 {
log.Printf("chat: leaving %d chat session(s) running on their harness for the next start", len(live))
}
return h.liveStore.RecordChatLiveSessions(ctx, live)
}
// reattachSessions is the start-up half of detachSessions: each recorded chat
// is observed again (stream, pending questions, background tasks) while its
// harness still has work, and a turn that ended while the server was down is
// folded in from the harness's history, so its reply still notifies and its
// queued prompts advance, as if it had been observed live.
func (h *chatHub) reattachSessions(ctx context.Context, broker chatBroker) {
if h.liveStore == nil || broker == nil {
return
}
live, err := h.liveStore.TakeChatLiveSessions(ctx)
if err != nil {
log.Printf("chat reattach: %v", err)
return
}
for _, session := range live {
if ctx.Err() != nil {
return
}
if !validChatSessionID(session.SessionID) {
continue
}
h.reattachSession(ctx, broker, session)
}
}
func (h *chatHub) reattachSession(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession) {
id := session.SessionID
running := false
if prober, ok := broker.(chatTurnProber); ok {
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
busy, err := prober.ProbeTurn(probeCtx, id)
cancel()
if err != nil {
// Unknown is not idle: keep observing rather than guess it ended.
log.Printf("chat reattach probe session=%s failed (%T)", id, err)
busy = session.Turn
}
running = busy
}
if running || session.Attention {
h.observeUnattended(id, broker)
}
if !session.Turn {
return
}
if !running {
h.foldEndedTurn(ctx, broker, session, nil)
return
}
h.mu.Lock()
room := h.rooms[id]
if room == nil {
h.mu.Unlock()
return
}
// The turn the previous process watched is still running: hold the queue
// behind it and remember it as started, so its end notifies as usual.
turn := h.turnLocked(id)
if turn.step == nil || turn.step.Status == protocol.ChatStepEnded {
turn.step = &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: session.NativeTurnID}
}
if room.nativeTurnID == "" {
room.nativeTurnID = session.NativeTurnID
}
room.observedTurn = true
h.mu.Unlock()
go h.awaitReattachedTurn(room, broker, session)
}
// awaitReattachedTurn waits for a reattached turn to end. The stream normally
// reports it; when the harness reports the turn idle without the stream having
// said so, the end is folded in from history.
func (h *chatHub) awaitReattachedTurn(room *chatRoom, broker chatBroker, session pipeline.ChatLiveSession) {
prober, _ := broker.(chatTurnProber)
started := h.clock()
ticker := time.NewTicker(chatReattachPoll)
defer ticker.Stop()
for {
select {
case <-room.ctx.Done():
return
case <-ticker.C:
}
h.mu.Lock()
ended := !room.observedTurn || h.rooms[room.sessionID] != room
h.mu.Unlock()
if ended || prober == nil || h.clock().Sub(started) >= h.unattended.turnLimit {
return
}
probeCtx, cancel := context.WithTimeout(room.ctx, 5*time.Second)
busy, err := prober.ProbeTurn(probeCtx, room.sessionID)
cancel()
if err != nil || busy {
continue
}
h.foldEndedTurn(room.ctx, broker, session, room)
return
}
}
// foldEndedTurn settles a turn that ended without the hub observing its end:
// attached clients are told to re-read the authoritative history, the room's
// turn slot is released (so queued prompts are delivered) and the reply is
// announced once, with its text read back from the harness.
func (h *chatHub) foldEndedTurn(ctx context.Context, broker chatBroker, session pipeline.ChatLiveSession, room *chatRoom) {
id := session.SessionID
readCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
page, err := broker.Messages(readCtx, id, 20, "")
cancel()
if err != nil {
log.Printf("chat reattach history session=%s failed (%T)", id, err)
}
replyID, body := lastReply(page.Messages)
h.mu.Lock()
defer h.mu.Unlock()
if room != nil && h.rooms[id] == room && room.observedTurn {
end := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepEnded, TurnID: room.nativeTurnID}}
h.applyLocked(end)
h.finishTurnLocked(room)
resync := protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventResync}
for _, sub := range room.subs {
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &resync})
enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &end})
}
if room.watching && len(room.subs) == 0 && len(room.queue) == 0 && !h.keepUnattendedLocked(room, false, false) {
room.watching = false
h.releaseRoomLocked(room)
}
}
if replyID == "" || !h.attentionWanted() {
return
}
key := session.NativeTurnID
if key == "" {
key = replyID
}
if !h.claimChatAttentionLocked("reply\x00" + id + "\x00" + key) {
return
}
if body == "" {
body = chatReplyFallbackBody
}
log.Printf("chat session %s: turn finished while the server was restarting; reply read back from history", id)
h.startChatAttention(chatAttention{sessionID: id, reply: true, body: body, id: chatNotificationID(id, "reply", key)})
}
// lastReply returns the id and bounded text of the assistant reply that ends
// a history page (messages after the last user message), or "" when the page
// ends with the user's own message.
func lastReply(messages []protocol.ChatMessage) (string, string) {
for i := len(messages) - 1; i >= 0; i-- {
message := messages[i]
if message.Role == protocol.ChatRoleUser {
return "", ""
}
if message.Kind != "" {
continue
}
var text strings.Builder
for _, part := range message.Parts {
if part.Type != protocol.ChatPartText || part.Text == "" {
continue
}
if text.Len() > 0 {
text.WriteString("\n\n")
}
text.WriteString(part.Text)
}
return message.ID, truncateRunes(paragraphs(text.String()), chatReplyBodyRunes)
}
return "", ""
}
+203
View File
@@ -0,0 +1,203 @@
package server
import (
"context"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// restartChat is a harness fake whose sessions live outside the server, the
// way an OpenCode service or the Codex App Server keeps them: a turn keeps
// running (probe) and its history (Messages) is readable after the server
// that started it is gone.
type restartChat struct {
*observedChat
hmu sync.Mutex
history []protocol.ChatMessage
}
func newRestartChat() *restartChat { return &restartChat{observedChat: newObservedChat()} }
func (c *restartChat) Messages(context.Context, string, int, string) (protocol.ChatHistoryPage, error) {
c.hmu.Lock()
defer c.hmu.Unlock()
return protocol.ChatHistoryPage{Messages: append([]protocol.ChatMessage(nil), c.history...)}, nil
}
func (c *restartChat) setProbe(running bool) {
c.smu.Lock()
c.probe["ses_1"] = running
c.smu.Unlock()
}
// chatRestartServer is one server process over a shared database.
type chatRestartServer struct {
ctx context.Context
cancel context.CancelFunc
srv *httptest.Server
handler *runHandler
}
func startChatRestartServer(t *testing.T, broker chatBroker, path string) *chatRestartServer {
t.Helper()
store := openQueueStore(t, path)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
handler := HandlerWithOptions(ctx, Options{Chat: broker, DetachSessions: true}, store).(*runHandler)
s := &chatRestartServer{ctx: ctx, cancel: cancel, srv: httptest.NewServer(handler), handler: handler}
t.Cleanup(s.stop)
return s
}
// shutdown is the server's SIGTERM path: record live chats, then stop.
func (s *chatRestartServer) shutdown() {
DetachSessions(s.handler)
s.stop()
WaitRuns(s.handler)
}
func (s *chatRestartServer) stop() {
s.srv.CloseClientConnections()
s.srv.Close()
s.cancel()
}
func restartChatDB(t *testing.T) string {
t.Helper()
path := filepath.Join(t.TempDir(), "chat.db")
seed := openQueueStore(t, path)
if err := seed.RecordOperatorSession(context.Background(), "ses_1", t.TempDir(), "Release prep", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
if _, err := seed.UpdateNotificationSettings(context.Background(), map[string]any{protocol.NotifyChatReplyFinished: true}); err != nil {
t.Fatal(err)
}
seed.Close()
return path
}
func fastReattach(t *testing.T) {
t.Helper()
fastUnattended(t)
t.Setenv("PIPELINES_NTFY_URL", "")
poll := chatReattachPoll
chatReattachPoll = 20 * time.Millisecond
t.Cleanup(func() { chatReattachPoll = poll })
}
// startTurnThenRestart prompts on the first server, holds the turn in flight
// (and optionally queues a second prompt), and shuts that server down.
func startTurnThenRestart(t *testing.T, path string, queueSecond bool) {
t.Helper()
first := newRestartChat()
server := startChatRestartServer(t, first, path)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "build it", "op-1")
first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
waitFor(t, "first prompt", func() bool { return first.promptCount() == 1 })
if queueSecond {
if ack := sendPrompt(t, server.ctx, conn, "p2", "then ship it", "op-2"); !ack.ChatQueued {
t.Fatalf("second prompt was not queued behind the running turn: %+v", ack)
}
}
server.shutdown()
}
func replyNotes(t *testing.T, path string) []protocol.AppNotification {
t.Helper()
return chatNotes(t, openQueueStore(t, path), protocol.NotificationChatReply)
}
// A turn in flight at shutdown that finished while the server was down is
// folded in from the harness's history: its reply notifies once, with the
// text the harness recorded during the downtime.
func TestChatTurnFinishedDuringRestartIsFoldedIn(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
startTurnThenRestart(t, path, false)
second := newRestartChat()
second.setProbe(false) // ended while the server was down
second.history = []protocol.ChatMessage{
{ID: "msg_u", SessionID: "ses_1", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "build it"}}},
{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Built while you were away."}}},
}
startChatRestartServer(t, second, path)
waitFor(t, "reply folded in from history", func() bool { return len(replyNotes(t, path)) == 1 })
if notes := replyNotes(t, path); !strings.Contains(notes[0].Message, "Built while you were away.") {
t.Fatalf("reply body not read back from history: %+v", notes[0])
}
}
// A turn still running after the restart keeps its queued prompt waiting;
// when the harness reports the turn idle (its end event was lost with the old
// stream), the end is folded in and the queued prompt is delivered.
func TestChatQueuedPromptDeliveredAfterRestart(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
startTurnThenRestart(t, path, true)
second := newRestartChat()
second.setProbe(true)
second.history = []protocol.ChatMessage{{ID: "msg_a", SessionID: "ses_1", Role: "assistant", Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "First part done."}}}}
startChatRestartServer(t, second, path)
waitFor(t, "reattached stream", func() bool { active, _ := second.streams(); return active >= 1 })
time.Sleep(2 * chatQueueRecoverySettle)
if second.promptCount() != 0 {
t.Fatalf("queued prompt delivered into a turn still running: %v", second.promptTexts())
}
second.setProbe(false)
waitFor(t, "queued prompt delivered", func() bool { return second.promptCount() == 1 })
if texts := second.promptTexts(); texts[0] != "then ship it" {
t.Fatalf("wrong prompt delivered after restart: %v", texts)
}
waitFor(t, "reply notified once", func() bool { return len(replyNotes(t, path)) == 1 })
}
// A question pending when the server stopped is observed again after the
// restart with no client attached, and a client that opens the chat sees it.
func TestChatPendingQuestionSurvivesRestart(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
question := protocol.ChatPermissionFrame{SessionID: "ses_1", RequestID: "req_1", Action: "bash", Resources: []string{"make release"}}
first := newRestartChat()
first.perm = []protocol.ChatPermissionFrame{question}
server := startChatRestartServer(t, first, path)
conn := dialChatConn(t, server.ctx, server.srv)
sendPrompt(t, server.ctx, conn, "p1", "release", "op-1")
first.push(stepFrameFor(protocol.ChatStepStarted, "turn_1"))
waitFor(t, "question surfaced", func() bool { return len(server.handler.hub.pendingFrames("ses_1")) == 1 })
server.shutdown()
second := newRestartChat()
second.setProbe(true)
second.perm = []protocol.ChatPermissionFrame{question}
restarted := startChatRestartServer(t, second, path)
waitFor(t, "question observed after restart", func() bool { return len(restarted.handler.hub.pendingFrames("ses_1")) == 1 })
// A client opening the chat after the restart is shown the question and
// its answer reaches the harness.
phone := dialChatConn(t, restarted.ctx, restarted.srv)
writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatSubscribe, ID: "s1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
readUntil(t, restarted.ctx, phone, protocol.ChatSubscribe)
if pending := nextPermission(t, restarted.ctx, phone, "req_1"); pending.Resolved {
t.Fatalf("question resolved by the restart: %+v", pending)
}
writeChat(t, restarted.ctx, phone, protocol.Message{Type: protocol.ChatReply, ID: "a1", ChatReply: &protocol.ChatReplyRequest{SessionID: "ses_1", RequestID: "req_1", Decision: protocol.ChatDecisionOnce, Client: "Phone"}})
readUntil(t, restarted.ctx, phone, protocol.ChatReply)
waitFor(t, "answer delivered to the harness", func() bool { return second.replyCount() == 1 })
}
// Nothing was in flight: nothing is recorded and nothing is observed.
func TestChatIdleShutdownRecordsNothing(t *testing.T) {
fastReattach(t)
path := restartChatDB(t)
server := startChatRestartServer(t, newRestartChat(), path)
server.shutdown()
live, err := openQueueStore(t, path).TakeChatLiveSessions(context.Background())
if err != nil || len(live) != 0 {
t.Fatalf("idle shutdown recorded live chats: %+v %v", live, err)
}
}
+3
View File
@@ -54,6 +54,9 @@ type chatHub struct {
// queueStore persists prompts queued behind a running turn; nil keeps the
// queue in memory only (tests and store-less servers).
queueStore chatPromptQueueStore
// liveStore records the chats a stopping server leaves live on their
// harness (chat_detach.go); nil reattaches nothing beyond the start probe.
liveStore chatLiveStore
// recentOps/recentOpOrder are the bounded record of accepted prompt
// operation ids, so a client retry is acknowledged instead of re-sent.
recentOps map[string]struct{}
+22
View File
@@ -183,6 +183,28 @@ func (m *chatManager) Close() {
closeHarness(claudeClient)
}
// DetachSessions is the shutdown counterpart of Close for a server that keeps
// chat sessions running across a restart (see chat_detach.go). Each harness
// that implements chatSessionDetacher releases its handles without stopping a
// session; one that does not is closed as before. The OpenCode connection's
// lifetime belongs to the caller (cmd/pipelines-server), so it is only detached.
func (m *chatManager) DetachSessions() {
m.mu.Lock()
m.closed = true
codexClient, claudeClient := m.codex, m.claude
m.mu.Unlock()
if detacher, ok := m.opencode.(chatSessionDetacher); ok {
detacher.DetachSessions()
}
for _, client := range []chatBroker{codexClient, claudeClient} {
if detacher, ok := client.(chatSessionDetacher); ok {
detacher.DetachSessions()
continue
}
closeHarness(client)
}
}
// Backends lists the harnesses available for a new session.
func (m *chatManager) Backends() []string {
backends := []string{}
+385
View File
@@ -0,0 +1,385 @@
package server
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// TestRestartKeepsRealOpenCodeSessions is the opt-in end-to-end restart check
// against the installed OpenCode binary and a real provider:
//
// PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m
//
// It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch
// database and a free loopback port (with only the operator's OpenCode
// provider login copied in), starts a chat turn and a one-stage run that each
// run `sleep 30`, restarts the server mid-turn, and checks both finish
// instead of being aborted. Every scratch process is killed afterwards.
func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
if os.Getenv("PIPELINES_RESTART_LIVE") != "1" {
t.Skip("set PIPELINES_RESTART_LIVE=1 to run the real-OpenCode restart check")
}
realHome, err := os.UserHomeDir()
if err != nil {
t.Fatal(err)
}
root, err := os.MkdirTemp("", "pipelines-restart-live-")
if err != nil {
t.Fatal(err)
}
t.Logf("scratch root %s", root)
home := filepath.Join(root, "home")
env := scratchEnvironment(t, realHome, home)
binary := filepath.Join(root, "pipelines-server")
build := exec.Command("go", "build", "-o", binary, "./cmd/pipelines-server")
build.Dir = filepath.Join("..", "..")
if out, err := build.CombinedOutput(); err != nil {
t.Fatalf("build: %v\n%s", err, out)
}
// Chat creation activates the manager skill, so install the current
// source's skills into the scratch harness roots.
skills := exec.Command("go", "run", "./cmd/pipelines", "skill", "install")
skills.Dir, skills.Env = filepath.Join("..", ".."), env
if out, err := skills.CombinedOutput(); err != nil {
t.Fatalf("skill install: %v\n%s", err, out)
}
db := filepath.Join(root, "data", "pipelines.db")
if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil {
t.Fatal(err)
}
store, err := pipeline.Open(db)
if err != nil {
t.Fatal(err)
}
if _, err := store.Submit(context.Background(), "restart-live", restartLiveStage); err != nil {
t.Fatal(err)
}
items, _ := store.List(context.Background())
store.Close()
addr := freeLoopback(t)
logPath := filepath.Join(root, "server.log")
server := startScratchServer(t, binary, db, addr, env, logPath)
t.Cleanup(func() {
server.stop()
killScratchServices(t, home)
})
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
defer cancel()
chatDir := filepath.Join(root, "chat-work")
runDir := filepath.Join(root, "run-work")
for _, dir := range []string{chatDir, runDir} {
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
}
client := dialLive(t, ctx, addr)
created := client.request(t, ctx, protocol.Message{Type: protocol.ChatCreate, ID: "c1", ChatCreate: &protocol.ChatCreateRequest{Title: "restart live", Directory: chatDir, OperationID: "create-1"}}, protocol.ChatCreate)
if created.ChatSession == nil {
t.Fatalf("chat_create = %+v", created)
}
sessionID := created.ChatSession.ID
prompted := time.Now()
client.request(t, ctx, protocol.Message{Type: protocol.ChatPrompt, ID: "p1", ChatPrompt: &protocol.ChatPromptRequest{SessionID: sessionID, OperationID: "op-1", Client: "restart-live",
Text: "Use your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then reply with exactly the text RESTART-CHAT-OK and nothing else."}}, protocol.ChatPrompt)
started := client.request(t, ctx, protocol.Message{Type: protocol.RunPipeline, ID: "r1", PipelineID: items[0].ID, Version: 1, RequestKey: "restart-live", Workspace: runDir, Input: map[string]any{"prompt": "restart live check"}}, protocol.RunStarted)
if started.RunDetail == nil {
t.Fatalf("run_pipeline = %+v", started)
}
runID := started.RunDetail.ID
// Wait until both harness sessions are executing the sleep.
waitLive(t, ctx, "chat shell running", func() bool { return chatShellStatus(t, ctx, addr, sessionID) == "running" })
var stageSession string
waitLive(t, ctx, "stage session admitted", func() bool {
d := getLiveRun(t, ctx, addr, runID)
if len(d.Steps) == 1 && d.Steps[0].SessionID != "" {
stageSession = d.Steps[0].SessionID
return true
}
return false
})
time.Sleep(8 * time.Second) // the stage's shell is inside its sleep
client.close()
t.Log("restarting the scratch server mid-turn")
server.stop()
server = startScratchServer(t, binary, db, addr, env, logPath)
var reply string
waitLiveFor(t, ctx, 5*time.Minute, "chat reply after restart", func() bool {
reply = chatReply(t, ctx, addr, sessionID)
return strings.Contains(reply, "RESTART-CHAT-OK")
})
if status := chatShellStatus(t, ctx, addr, sessionID); status != "completed" {
t.Fatalf("chat shell was not completed after the restart: %q", status)
}
if elapsed := time.Since(prompted); elapsed < 30*time.Second {
t.Fatalf("chat reply arrived before the sleep could finish (%s): the turn was cut short", elapsed)
}
var final *protocol.RunDetail
waitLiveFor(t, ctx, 6*time.Minute, "run finished after restart", func() bool {
final = getLiveRun(t, ctx, addr, runID)
return final.FinishedAt != ""
})
if final.Status != "succeeded" || len(final.Steps) != 1 || final.Steps[0].Status != "completed" || final.Steps[0].SessionID != stageSession {
t.Fatalf("run did not continue on its own session: status=%s diagnostic=%q steps=%+v", final.Status, final.Diagnostic, final.Steps)
}
logData, _ := os.ReadFile(logPath)
for _, want := range []string{"leaving 1 chat session(s) running", "left running for the next start to reattach", "reattached stage one"} {
if !bytes.Contains(logData, []byte(want)) {
t.Errorf("server log lacks %q", want)
}
}
t.Logf("chat reply %q; run %d %s on session %s", reply, runID, final.Status, stageSession)
}
const restartLiveStage = "# stage one\n\n```prompt\nUse your shell tool to run exactly this command once: sleep 30 . Wait for it to finish. Then write the turn output: a short body saying RESTART-STAGE-OK, with the outcome success.\n```\n\n```tools\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\ntimeout=15m\n```\n"
// scratchEnvironment is the server's environment: a private HOME/XDG tree with
// only the operator's OpenCode provider login and a minimal configuration.
func scratchEnvironment(t *testing.T, realHome, home string) []string {
t.Helper()
config, data, cache := filepath.Join(home, ".config"), filepath.Join(home, ".local", "share"), filepath.Join(home, ".cache")
for _, dir := range []string{filepath.Join(config, "opencode"), filepath.Join(data, "opencode"), cache} {
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
}
auth, err := os.ReadFile(filepath.Join(realHome, ".local", "share", "opencode", "auth.json"))
if err != nil {
t.Skipf("no OpenCode login to copy: %v", err)
}
if err := os.WriteFile(filepath.Join(data, "opencode", "auth.json"), auth, 0600); err != nil {
t.Fatal(err)
}
// A cold model catalog can leave the first service unable to resolve the
// model; seed the public catalog cache (not a credential).
if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil {
if err := os.MkdirAll(filepath.Join(cache, "opencode"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(cache, "opencode", "models.json"), catalog, 0600); err != nil {
t.Fatal(err)
}
}
settings := `{"$schema":"https://opencode.ai/config.json","model":"opencode-go/deepseek-v4.1-flash","permissions":[{"action":"shell","resource":"*","effect":"allow"}]}`
if err := os.WriteFile(filepath.Join(config, "opencode", "opencode.json"), []byte(settings), 0600); err != nil {
t.Fatal(err)
}
var env []string
for _, entry := range os.Environ() {
key, _, _ := strings.Cut(entry, "=")
switch {
case key == "HOME", strings.HasPrefix(key, "XDG_"), strings.HasPrefix(key, "PIPELINES_"), strings.HasPrefix(key, "OPENCODE_"):
continue
}
env = append(env, entry)
}
return append(env, "HOME="+home, "XDG_CONFIG_HOME="+config, "XDG_DATA_HOME="+data, "XDG_CACHE_HOME="+cache, "XDG_STATE_HOME="+filepath.Join(home, ".local", "state"),
"PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off")
}
func opencodePath() string {
if path, err := exec.LookPath("opencode"); err == nil {
return path
}
return "/opt/homebrew/bin/opencode"
}
type scratchServer struct {
t *testing.T
cmd *exec.Cmd
}
func startScratchServer(t *testing.T, binary, db, addr string, env []string, logPath string) *scratchServer {
t.Helper()
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
t.Fatal(err)
}
cmd := exec.Command(binary, "-db", db, "-addr", addr, "-mdns=false")
cmd.Env, cmd.Stdout, cmd.Stderr = env, logFile, logFile
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
logFile.Close()
s := &scratchServer{t: t, cmd: cmd}
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
if resp, err := http.Get("http://" + addr + "/healthz"); err == nil {
resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return s
}
}
time.Sleep(200 * time.Millisecond)
}
s.stop()
data, _ := os.ReadFile(logPath)
t.Fatalf("scratch server did not become healthy:\n%s", data)
return nil
}
// stop is a service stop/restart: SIGTERM, then wait for a clean exit.
func (s *scratchServer) stop() {
if s == nil || s.cmd == nil || s.cmd.Process == nil || s.cmd.ProcessState != nil {
return
}
_ = s.cmd.Process.Signal(syscall.SIGTERM)
done := make(chan struct{})
go func() { _ = s.cmd.Wait(); close(done) }()
select {
case <-done:
case <-time.After(60 * time.Second):
_ = s.cmd.Process.Kill()
<-done
s.t.Error("scratch server did not stop within 60s of SIGTERM")
}
}
// killScratchServices stops the detached OpenCode services the scratch server
// owned (their pids are in its private registrations).
func killScratchServices(t *testing.T, home string) {
for _, name := range []string{"opencode-stage-service.json", "opencode-chat-service.json"} {
data, err := os.ReadFile(filepath.Join(home, "Library", "Caches", "pipelines", "private", name))
if err != nil {
data, err = os.ReadFile(filepath.Join(home, ".cache", "pipelines", "private", name))
}
if err != nil {
continue
}
var state struct{ PID int }
if json.Unmarshal(data, &state) == nil && state.PID > 0 {
_ = syscall.Kill(state.PID, syscall.SIGTERM)
time.Sleep(time.Second)
_ = syscall.Kill(state.PID, syscall.SIGKILL)
t.Logf("stopped scratch %s (pid %d)", strings.TrimSuffix(name, ".json"), state.PID)
}
}
}
func freeLoopback(t *testing.T) string {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
return "127.0.0.1:" + strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)
}
type liveClient struct{ conn *websocket.Conn }
func dialLive(t *testing.T, ctx context.Context, addr string) *liveClient {
t.Helper()
conn, _, err := websocket.Dial(ctx, "ws://"+addr+"/connect", nil)
if err != nil {
t.Fatal(err)
}
conn.SetReadLimit(16 << 20)
return &liveClient{conn: conn}
}
func (c *liveClient) close() { c.conn.CloseNow() }
func (c *liveClient) request(t *testing.T, ctx context.Context, msg protocol.Message, want string) protocol.Message {
t.Helper()
if err := wsjson.Write(ctx, c.conn, msg); err != nil {
t.Fatal(err)
}
read, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
for {
var reply protocol.Message
if err := wsjson.Read(read, c.conn, &reply); err != nil {
t.Fatalf("waiting for %s: %v", want, err)
}
if reply.ID != msg.ID {
continue
}
if reply.Type == protocol.Error {
t.Fatalf("%s refused: %s", msg.Type, reply.Error)
}
if reply.Type == want {
return reply
}
}
}
func liveQuery(t *testing.T, ctx context.Context, addr string, msg protocol.Message, want string) protocol.Message {
t.Helper()
c := dialLive(t, ctx, addr)
defer c.close()
return c.request(t, ctx, msg, want)
}
func getLiveRun(t *testing.T, ctx context.Context, addr string, id int64) *protocol.RunDetail {
t.Helper()
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.GetRun, ID: "g", RunID: id}, protocol.Run)
if reply.RunDetail == nil {
t.Fatalf("get_run = %+v", reply)
}
return reply.RunDetail
}
func liveMessages(t *testing.T, ctx context.Context, addr, session string) []protocol.ChatMessage {
t.Helper()
reply := liveQuery(t, ctx, addr, protocol.Message{Type: protocol.ChatMessages, ID: "m", ChatQuery: &protocol.ChatQuery{SessionID: session, Limit: 50}}, protocol.ChatMessages)
if reply.ChatHistory != nil {
return reply.ChatHistory.Messages
}
return reply.ChatMessages
}
// chatShellStatus reports the status of the newest shell tool call.
func chatShellStatus(t *testing.T, ctx context.Context, addr, session string) string {
status := ""
for _, message := range liveMessages(t, ctx, addr, session) {
for _, part := range message.Parts {
if part.Type == protocol.ChatPartTool && part.State != nil && strings.Contains(strings.ToLower(part.Name+string(part.State.Input)), "sleep") {
status = part.State.Status
}
}
}
return status
}
func chatReply(t *testing.T, ctx context.Context, addr, session string) string {
_, text := lastReply(liveMessages(t, ctx, addr, session))
return text
}
func waitLive(t *testing.T, ctx context.Context, what string, check func() bool) {
t.Helper()
waitLiveFor(t, ctx, 3*time.Minute, what, check)
}
func waitLiveFor(t *testing.T, ctx context.Context, limit time.Duration, what string, check func() bool) {
t.Helper()
deadline := time.Now().Add(limit)
for time.Now().Before(deadline) && ctx.Err() == nil {
if check() {
return
}
time.Sleep(2 * time.Second)
}
t.Fatal(fmt.Sprintf("timed out waiting for %s", what))
}
+220
View File
@@ -0,0 +1,220 @@
package server
import (
"context"
"errors"
"fmt"
"log"
"time"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// Runs survive a server restart. With detach enabled (the installed server),
// a stopping server cancels each run's context with stage.ErrServerShutdown:
// stage clients leave their admitted harness session running and owned, and the
// loop returns without settling anything. The next process plans, before it
// serves requests, how to continue every run it finds in flight:
//
// - a running visit with a live OpenCode V2 session is re-adopted: the loop
// re-enters that visit, attaches to the session, waits for it to settle (or
// reads the result it reached while the server was down) and continues the
// graph as if it had watched it;
// - a visit begun but never handed to a harness is marked interrupted and the
// same stage starts again (nothing was admitted, so nothing replays);
// - a run between visits continues with its next stage;
// - a session whose harness process is gone, or that belongs to a harness the
// server owned as a child (Pi, OpenCode V1), falls back to the previous
// behaviour: the run is interrupted, with the reason in its diagnostic.
//
// Only an operator cancel (cancel_run) stops a stage session.
type reattachKind int
const (
reattachAdopt reattachKind = iota
reattachRestart
reattachBetween
)
// reattachPlan is one run start-up recovery kept for reattachRuns.
type reattachPlan struct {
runID int64
kind reattachKind
// step is the running visit (adopt/restart) or the last completed one.
step protocol.RunStep
owner pipeline.SessionOwner
}
// adoptedVisit is the visit a runLoop re-adopts instead of starting.
type adoptedVisit struct {
sequence int
owner pipeline.SessionOwner
}
// stageServiceAlive reports whether a retained stage session's harness service
// is still running. It is a seam for tests.
var stageServiceAlive = func(ctx context.Context, owner pipeline.SessionOwner) error {
return opencode.ReferenceAlive(ctx, owner.Reference)
}
// interruptRecoveredSession stops a previously active session that start-up
// recovery does not re-adopt. It is a seam for tests.
var interruptRecoveredSession = func(ctx context.Context, owner pipeline.SessionOwner) error {
if owner.Backend == "pi" {
return stage.InterruptPi(ctx, owner.Reference)
}
return (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(ctx, owner.ID)
}
// runContext derives one run's execution context. With detach, a server
// shutdown cancels it with stage.ErrServerShutdown as the cause, while the
// returned cancel (an operator cancel or the run's own end) uses the ordinary
// context.Canceled, so stage clients can tell the two apart.
func (r *runner) runContext() (context.Context, context.CancelFunc) {
if !r.detach {
return context.WithCancel(r.ctx)
}
ctx, cancel := context.WithCancelCause(context.WithoutCancel(r.ctx))
stop := context.AfterFunc(r.ctx, func() { cancel(stage.ErrServerShutdown) })
return ctx, func() { stop(); cancel(context.Canceled) }
}
// planReattach decides, per in-flight run, whether start-up continues it.
// Runs it does not plan keep today's recovery (interrupted); reasons carries the
// explanation for the ones that could not be reattached.
// gone lists runs whose session's service is not running: there is nothing
// left to interrupt, so recovery only clears their ownership.
func planReattach(ctx context.Context, s *pipeline.Store) ([]reattachPlan, map[int64]string, map[int64]bool, error) {
reasons, gone := map[int64]string{}, map[int64]bool{}
rows, err := s.ListRuns(ctx)
if err != nil {
return nil, nil, nil, err
}
var plans []reattachPlan
for _, row := range rows {
if row.Status != protocol.RunStatusRunning || row.CancelRequested || row.CleanupUnconfirmed {
continue
}
detail, err := s.GetRun(ctx, row.ID)
if err != nil {
return nil, nil, nil, err
}
if len(detail.Steps) == 0 {
continue
}
last := detail.Steps[0]
for _, step := range detail.Steps {
if step.Sequence > last.Sequence {
last = step
}
}
switch last.Status {
case protocol.StepStatusRunning:
if last.SessionID == "" {
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachRestart, step: last})
continue
}
owner, err := s.RunSessionOwner(ctx, row.ID, last.Sequence)
if err != nil {
return nil, nil, nil, err
}
owner.RunID = row.ID
if owner.Backend != "v2" {
name := "OpenCode V1"
if owner.Backend == "pi" {
name = "Pi"
}
reasons[row.ID] = fmt.Sprintf("%s stage sessions belong to the stopped server process, so stage %s could not be reattached", name, last.Stage)
continue
}
probe, cancel := context.WithTimeout(ctx, 5*time.Second)
aliveErr := stageServiceAlive(probe, owner)
cancel()
if aliveErr != nil {
gone[row.ID] = errors.Is(aliveErr, opencode.ErrServiceNotRunning)
reasons[row.ID] = fmt.Sprintf("the OpenCode service running stage %s was gone (%v), so its session could not be reattached", last.Stage, aliveErr)
continue
}
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachAdopt, step: last, owner: owner})
case "completed":
plans = append(plans, reattachPlan{runID: row.ID, kind: reattachBetween, step: last})
}
}
return plans, reasons, gone, nil
}
// reattachRuns continues every run start-up recovery kept. It is called once,
// after the runner's collaborators are wired and before requests are served.
func (r *runner) reattachRuns() {
plans := r.reattach
r.reattach = nil
for _, plan := range plans {
r.reattachRun(plan)
}
}
func (r *runner) reattachRun(plan reattachPlan) {
id := plan.runID
ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
defer cancel()
if plan.kind == reattachRestart {
if err := r.store.RestartStep(ctx, id, plan.step.Sequence, "Server restarted before this stage handed work to a harness; the stage was started again"); err != nil {
log.Printf("run %d: restart stage %s after server restart: %v", id, plan.step.Stage, err)
}
}
detail, err := r.store.GetRun(ctx, id)
if err != nil {
log.Printf("run %d: read for reattach: %v", id, err)
return
}
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
workspace := detail.Workspace
request := protocol.Message{WritePaths: detail.WritePaths, AllowTools: detail.AllowTools, Model: detail.Model, StageModels: detail.StageModels}
r.mu.Lock()
defer r.mu.Unlock()
switch plan.kind {
case reattachBetween:
log.Printf("run %d: continuing after stage %s following a server restart", id, plan.step.Stage)
r.launch(id, workspace, func(ctx context.Context) { r.resume(ctx, id, detail, workspace, plan.step) })
default:
state := rebuildAdoptState(detail, plan)
if plan.kind == reattachAdopt {
log.Printf("run %d: reattached stage %s #%d session %s after a server restart", id, plan.step.Stage, plan.step.Sequence, plan.owner.ID)
} else {
log.Printf("run %d: starting stage %s again after a server restart", id, plan.step.Stage)
}
r.launch(id, workspace, func(ctx context.Context) {
r.runLoop(ctx, id, detail.Graph, detail.Name, workspace, request, state)
})
}
}
// rebuildAdoptState restores the loop state at the planned visit: memory and
// visit counts from every other persisted step, the handoff input from the
// visit before it, and (for an adopted visit) the session to re-adopt.
func rebuildAdoptState(detail *protocol.RunDetail, plan reattachPlan) *runState {
decided := protocol.RunStep{Target: plan.step.Stage, Output: detail.Input}
previous := 0
for _, step := range detail.Steps {
if step.Sequence < plan.step.Sequence && step.Sequence > previous {
previous = step.Sequence
decided = step
decided.Target = plan.step.Stage
}
}
state := rebuildRunState(detail, decided)
state.key = plan.step.Stage
// The interrupted or adopted visit was counted; the loop counts it again.
if state.visits[plan.step.Stage] > 0 {
state.visits[plan.step.Stage]--
}
if plan.kind == reattachAdopt {
state.nextSequence = plan.step.Sequence
state.adopt = &adoptedVisit{sequence: plan.step.Sequence, owner: plan.owner}
}
return state
}
+262
View File
@@ -0,0 +1,262 @@
package server
import (
"context"
"fmt"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// restartFixture runs a two-stage pipeline on a detaching runner, stops that
// "server" while stage one is in flight, and reopens the store for a second
// runner, the way a restart does.
type restartFixture struct {
path, root, workspace string
runID int64
}
// stopMidStage starts the run and shuts the first runner down while stage one
// is executing. firstStage is the first process's executor for stage one; it
// must block until its context ends.
func stopMidStage(t *testing.T, firstStage func(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error)) restartFixture {
t.Helper()
f := restartFixture{path: filepath.Join(t.TempDir(), "restart.db"), root: t.TempDir(), workspace: t.TempDir()}
s, err := pipeline.Open(f.path)
if err != nil {
t.Fatal(err)
}
ctx, shutdown := context.WithCancel(context.Background())
if _, err := s.Submit(ctx, "restart", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunnerReattaching(ctx, s, f.root, true)
if err != nil {
t.Fatal(err)
}
r.exportSession = validGateExport
entered := make(chan struct{})
var id atomic.Int64
r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "one" {
t.Errorf("the first process reached stage %s", def.Key)
return stage.Result{}, fmt.Errorf("unexpected stage")
}
for id.Load() == 0 {
time.Sleep(time.Millisecond)
}
close(entered)
return firstStage(ctx, s, id.Load())
}
d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "restart", Workspace: f.workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
id.Store(d.ID)
f.runID = d.ID
<-entered
shutdown() // SIGTERM
r.wg.Wait()
r.schedulerWG.Wait()
persisted, err := s.GetRun(context.Background(), f.runID)
if err != nil {
t.Fatal(err)
}
if persisted.Status != protocol.RunStatusRunning || len(persisted.Steps) != 1 || persisted.Steps[0].Status != protocol.StepStatusRunning {
t.Fatalf("shutdown settled the run instead of leaving it in flight: %+v", persisted)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
return f
}
// restart opens the second process's runner on the same database.
func (f restartFixture) restart(t *testing.T, execute func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error)) (*runner, *pipeline.Store) {
t.Helper()
s, err := pipeline.Open(f.path)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunnerReattaching(ctx, s, f.root, true)
if err != nil {
cancel()
s.Close()
t.Fatal(err)
}
t.Cleanup(func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() })
r.exportSession = validGateExport
r.execute = func(ctx context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
return execute(ctx, s, def)
}
return r, s
}
func seamRestartRecovery(t *testing.T, alive error) *atomic.Int32 {
t.Helper()
interrupts := &atomic.Int32{}
previousAlive, previousInterrupt := stageServiceAlive, interruptRecoveredSession
stageServiceAlive = func(context.Context, pipeline.SessionOwner) error { return alive }
interruptRecoveredSession = func(context.Context, pipeline.SessionOwner) error { interrupts.Add(1); return nil }
t.Cleanup(func() { stageServiceAlive, interruptRecoveredSession = previousAlive, previousInterrupt })
return interrupts
}
// ownedLiveStage is a first-process stage one that admitted session
// ses_live (recorded like a real client does) and is still running at shutdown.
func ownedLiveStage(ctx context.Context, s *pipeline.Store, id int64) (stage.Result, error) {
if err := s.RunSession(context.Background(), id, 1, "ses_live", true, "v2", "ref-live"); err != nil {
return stage.Result{}, err
}
<-ctx.Done()
if stage.Detached(ctx) {
return stage.Result{}, stage.ErrDetached
}
return stage.Result{}, ctx.Err()
}
// TestStageRunContinuesToNextStageAfterRestart: the stage in flight at a server
// restart is re-adopted (never re-prompted, never interrupted), its result is
// folded in, and the run continues with the next stage to success.
func TestStageRunContinuesToNextStageAfterRestart(t *testing.T) {
interrupts := seamRestartRecovery(t, nil)
f := stopMidStage(t, ownedLiveStage)
var resumed atomic.Value
var secondRan atomic.Int32
r, s := f.restart(t, func(ctx context.Context, s *pipeline.Store, def stage.Definition) (stage.Result, error) {
if def.Key == "one" {
resumed.Store(stage.TakeResumeSession(ctx))
// The real client releases ownership once the session settled.
if err := s.RunSession(context.Background(), f.runID, 1, "ses_live", false, "v2", "ref-live"); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "finished while the server was down"}}, nil
}
secondRan.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next stage"}}, nil
})
if interrupts.Load() != 0 {
t.Fatal("start-up recovery interrupted a session that survived the restart")
}
r.reattachRuns()
r.wg.Wait()
d, err := s.GetRun(context.Background(), f.runID)
if err != nil {
t.Fatal(err)
}
if got, _ := resumed.Load().(string); got != "ses_live" {
t.Fatalf("stage one was not re-adopted from its session: %q", got)
}
if d.Status != "succeeded" || len(d.Steps) != 2 || secondRan.Load() != 1 {
t.Fatalf("run did not continue to the next stage: %+v", d)
}
one := d.Steps[0]
if one.Sequence != 1 || one.Status != "completed" || one.Output["markdown"] != "finished while the server was down" {
t.Fatalf("adopted visit not folded in: %+v", one)
}
if one.FileChanges == nil || one.FileChanges.Status != "partial" || !strings.Contains(one.FileChanges.Diagnostic, "restart") {
t.Fatalf("adopted visit claims a complete change capture: %+v", one.FileChanges)
}
}
// TestStageRunWithHarnessGoneFallsBackWithReason: when the harness process
// is gone after the restart, the run is interrupted as before, naming why.
func TestStageRunWithHarnessGoneFallsBackWithReason(t *testing.T) {
interrupts := seamRestartRecovery(t, fmt.Errorf("%w at http://127.0.0.1:1", opencode.ErrServiceNotRunning))
f := stopMidStage(t, ownedLiveStage)
r, s := f.restart(t, func(context.Context, *pipeline.Store, stage.Definition) (stage.Result, error) {
t.Error("a run whose harness was gone executed again")
return stage.Result{}, nil
})
r.reattachRuns()
r.wg.Wait()
if interrupts.Load() != 0 {
t.Fatal("recovery tried to interrupt a session on a service that is gone")
}
d, err := s.GetRun(context.Background(), f.runID)
if err != nil {
t.Fatal(err)
}
if d.Status != "interrupted" || !strings.Contains(d.Diagnostic, "was gone") || !strings.Contains(d.Steps[0].Diagnostic, "was gone") {
t.Fatalf("harness-gone fallback lacks a clear reason: status=%s run=%q step=%q", d.Status, d.Diagnostic, d.Steps[0].Diagnostic)
}
if owners, _ := s.ActiveSessionOwners(context.Background()); len(owners) != 0 {
t.Fatalf("a gone session is still recorded as active: %+v", owners)
}
}
// TestStageNotYetAdmittedStartsAgainAfterRestart: a visit the stopped server
// had begun but not handed to any session is started again, nothing replayed.
func TestStageNotYetAdmittedStartsAgainAfterRestart(t *testing.T) {
seamRestartRecovery(t, nil)
f := stopMidStage(t, func(ctx context.Context, _ *pipeline.Store, _ int64) (stage.Result, error) {
<-ctx.Done()
return stage.Result{}, ctx.Err()
})
var resumed atomic.Value
resumed.Store("")
r, s := f.restart(t, func(ctx context.Context, _ *pipeline.Store, def stage.Definition) (stage.Result, error) {
if def.Key == "one" {
resumed.Store(stage.TakeResumeSession(ctx))
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": def.Key}}, nil
})
r.reattachRuns()
r.wg.Wait()
d, _ := s.GetRun(context.Background(), f.runID)
if d.Status != "succeeded" || len(d.Steps) != 3 || d.Steps[0].Status != "interrupted" || d.Steps[1].Stage != "one" || d.Steps[1].Status != "completed" {
t.Fatalf("unadmitted stage was not started again: %+v", d)
}
if got := resumed.Load().(string); got != "" {
t.Fatalf("a fresh visit was treated as a re-adoption of %q", got)
}
}
// TestOperatorCancelStillStopsDetachingRun: only shutdown detaches; an
// operator cancel on a detaching server still settles the run as canceled.
func TestOperatorCancelStillStopsDetachingRun(t *testing.T) {
s, err := pipeline.Open(filepath.Join(t.TempDir(), "cancel.db"))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
if _, err := s.Submit(ctx, "cancel", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
r, err := newRunnerReattaching(ctx, s, t.TempDir(), true)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); s.Close() }()
entered := make(chan struct{})
detached := &atomic.Bool{}
r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
close(entered)
<-ctx.Done()
detached.Store(stage.Detached(ctx))
return stage.Result{}, ctx.Err()
}
d, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "cancel", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
<-entered
if err := r.cancel(d.ID); err != nil {
t.Fatal(err)
}
r.wg.Wait()
final, _ := s.GetRun(context.Background(), d.ID)
if detached.Load() || final.Status != "canceled" {
t.Fatalf("operator cancel did not stop the run: detached=%v %+v", detached.Load(), final)
}
}
+90 -23
View File
@@ -84,6 +84,10 @@ type runner struct {
// sizeKick wakes the background run-size backfill (sizeLoop); nil when the
// loop is not running (a runner built directly in a test).
sizeKick chan struct{}
// detach leaves harness sessions running when the server stops; reattach
// holds the runs start-up recovery kept for reattachRuns to continue.
detach bool
reattach []reattachPlan
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
@@ -96,6 +100,14 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
// starts and is never written afterwards, so those goroutines read it without
// locking.
func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) {
return newRunnerReattaching(ctx, s, artifactsRoot, false)
}
// newRunnerReattaching builds the runner. With detach set, a server shutdown
// leaves stage sessions running (see stage.ErrServerShutdown) and start-up
// recovery re-adopts the runs a previous process left in flight instead of
// interrupting them; reattachRuns then continues them.
func newRunnerReattaching(ctx context.Context, s *pipeline.Store, artifactsRoot string, detach bool) (*runner, error) {
if err := s.AcquireRunnerLock(); err != nil {
return nil, err
}
@@ -109,19 +121,30 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (
if err = s.BindLegacyScheduleWorkspace(ctx, workspace); err != nil {
return nil, err
}
var plans []reattachPlan
keep, reasons, gone := map[int64]bool{}, map[int64]string{}, map[int64]bool{}
if detach {
if plans, reasons, gone, err = planReattach(ctx, s); err != nil {
return nil, err
}
for _, plan := range plans {
keep[plan.runID] = true
}
}
sessions, err := s.ActiveSessionOwners(ctx)
if err != nil {
return nil, err
}
for _, owner := range sessions {
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
var err error
if owner.Backend == "pi" {
err = stage.InterruptPi(stop, owner.Reference)
} else {
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID)
if keep[owner.RunID] {
continue // Re-adopted below: never interrupt work that survived.
}
var err error
if !gone[owner.RunID] {
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
err = interruptRecoveredSession(stop, owner)
cancel()
}
cancel()
if err != nil {
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
}
@@ -129,7 +152,7 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (
return nil, err
}
}
if err := s.RecoverRuns(ctx); err != nil {
if err := s.RecoverRunsKeeping(ctx, keep, reasons); err != nil {
return nil, err
}
root := artifactsRoot
@@ -146,7 +169,7 @@ func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (
active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace}
}
}
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession, detach: detach, reattach: plans}
if err := r.startSessionExportJobs(); err != nil {
return nil, err
}
@@ -341,7 +364,7 @@ func (r *runner) startWithSchedule(request protocol.Message, origin *pipeline.Sc
// the registration when it returns. A fresh run and a resumed decision both go
// through it, so registration and teardown live once. Callers must hold r.mu.
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
ctx, cancel := context.WithCancel(r.ctx)
ctx, cancel := r.runContext()
generation := new(byte)
r.active[id] = activeRun{cancel: cancel, workspace: workspace, generation: generation}
r.wg.Add(1)
@@ -980,6 +1003,9 @@ type runState struct {
// have been resolved. It is intentionally not part of run input or memory.
retryOverride stage.Values
priorBudgets map[string]stage.BudgetSnapshot
// adopt names the visit to re-adopt from a previous server process instead
// of starting it; it is consumed by the loop's first iteration.
adopt *adoptedVisit
}
func newRunState(key string, input stage.Values) *runState {
@@ -1528,6 +1554,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
finish := func(status, outcome, diagnostic string) {
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
}
// canceled settles an operator cancel. A server shutdown is not a cancel:
// the run is left exactly as persisted for the next process to continue.
canceled := func() {
if stage.Detached(ctx) {
log.Printf("run %d: server stopping; run left in flight for the next start to continue", id)
return
}
finish("canceled", "", ctx.Err().Error())
}
defer func() {
if recovered := recover(); recovered != nil {
finish("failed", "error", "stage execution panicked")
@@ -1554,7 +1589,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
for sequence := state.nextSequence; sequence <= 30; sequence++ {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
canceled()
return
}
key := state.key
@@ -1600,8 +1635,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
return
}
}
// adopted is the visit a previous server process began and left running
// across its restart: its step row, turn files and session already exist.
adopted := state.adopt
if adopted != nil && adopted.sequence != sequence {
adopted = nil
}
state.adopt = nil
_, declaresSkip := def.Edges["skipped"]
skip := !isHuman && declaresSkip && autoSkipEligible(def.Configs, state)
skip := adopted == nil && !isHuman && declaresSkip && autoSkipEligible(def.Configs, state)
state.nextSequence = sequence + 1
state.visits[key]++
// Publish the run's server-captured changed paths into the input memory
@@ -1643,13 +1685,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
retryInput = input
state.retryOverride = nil
}
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
if adopted == nil {
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
if ctx.Err() != nil {
canceled()
return
}
finish("failed", "error", err.Error())
return
}
finish("failed", "error", err.Error())
return
}
// queued -> running: report the new current stage and the step start.
r.emitRun(protocol.RunEventFrame{
@@ -1668,7 +1712,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
r.mu.Unlock()
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
canceled()
return
}
finish("failed", "error", err.Error())
@@ -1744,9 +1788,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
var publicBefore map[string]bool
if stage.EffectiveType(def.Type) == stage.ModelType {
visitTurn = r.turnPaths(id, sequence, key)
if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil {
finish("failed", "error", turnErr.Error())
return
if adopted == nil {
if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil {
finish("failed", "error", turnErr.Error())
return
}
}
artifactsDir = visitTurn.TurnDir
def.ArtifactsDir = artifactsDir
@@ -1779,6 +1825,11 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
var piClient *stage.PiClient
var err error
stageCtx := ctx
if adopted != nil {
// Attach to the surviving session instead of prompting again.
client.Backend, client.Reference = adopted.owner.Backend, adopted.owner.Reference
stageCtx = stage.WithResumeSession(stageCtx, adopted.owner.ID)
}
var modelConfig stage.ModelConfig
var budgetBefore stage.BudgetSnapshot
if stage.EffectiveType(def.Type) == stage.ModelType {
@@ -1846,6 +1897,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
stageSession = client.SessionID
} else if stage.EffectiveType(def.Type) == stage.ModelType {
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
if adopted != nil {
plain.Backend, plain.Reference = adopted.owner.Backend, adopted.owner.Reference
}
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -1860,6 +1914,13 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
}
}()
if r.detach && stage.Detached(ctx) {
// The server is stopping. The stage left its session running and
// owned (or never admitted one); nothing about this visit is settled
// here. The next process re-adopts the session and continues.
log.Printf("run %d stage %s #%d: server stopping; stage left running for the next start to reattach", id, key, sequence)
return
}
ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second)
activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id)
ownershipCancel()
@@ -1877,6 +1938,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
after := filediff.Capture(snapshotCtx, workspace, before)
snapshotCancel()
changes := filediff.Compare(before, after)
if adopted != nil && changes.Status == "complete" {
// The pre-visit snapshot belonged to the stopped process; only the
// changes made after reattaching were compared.
changes.Status = "partial"
changes.Diagnostic = "Server restarted during this stage; changes made before the restart are not listed."
}
if errors.Is(err, stage.ErrStageCleanup) {
changes.Status = "partial"
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
@@ -2062,7 +2129,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
return
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
canceled()
return
}
// Advance the in-memory UI-surface evidence only after the step is
@@ -2099,7 +2166,7 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
},
}, false)
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
canceled()
return
}
if terminal {
+41 -7
View File
@@ -58,6 +58,12 @@ type Options struct {
// target-aware verification. Definitions can name only a registered ID; they
// cannot select an endpoint or supply provenance.
VerificationFixtures []VerificationFixture
// DetachSessions keeps harness sessions running across a server restart:
// a stopping server leaves stage and chat sessions running (DetachSessions
// on the handler records the chats) and the next start reattaches them.
// False keeps the previous behaviour (tests, embedded servers): a stopping
// server interrupts its runs and the next start marks them interrupted.
DetachSessions bool
}
// Handler serves the persistent connection with default options. Canceling ctx
@@ -77,7 +83,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
var runs *runner
var runErr error
if len(stores) > 0 {
runs, runErr = newRunnerAt(ctx, stores[0], options.ArtifactsRoot)
runs, runErr = newRunnerReattaching(ctx, stores[0], options.ArtifactsRoot, options.DetachSessions)
if runErr == nil {
runs.events = events
}
@@ -115,15 +121,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
settingsStore = stores[0]
}
hub.settings = settingsStore
if options.Chat != nil && registry != nil {
// Keep observing chats whose turn a previous process left running, so
// their reply or question still notifies with no client attached.
go hub.observeRunningTurns(ctx, options.Chat)
}
if len(stores) > 0 {
// Prompts queued behind a running turn are durable and owned by their
// session; resume delivering any a previous process left queued.
hub.queueStore = stores[0]
hub.liveStore = stores[0]
if options.Chat != nil {
go hub.resumeQueued(ctx, options.Chat)
}
@@ -449,12 +451,44 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
}
mux.HandleFunc("GET /connect", connect)
return &runHandler{Handler: mux, runner: runs}
if options.Chat != nil && registry != nil {
// Reattach the chats a previous process left live (and fold in turns
// that ended while it was down), then keep observing any other chat
// whose turn is still running, so a reply or question still notifies
// with no client attached.
go func() {
hub.reattachSessions(ctx, options.Chat)
hub.observeRunningTurns(ctx, options.Chat)
}()
}
if runs != nil {
// Continue the runs a previous process left in flight, now that the
// runner's collaborators are wired and before any request is served.
runs.reattachRuns()
}
return &runHandler{Handler: mux, runner: runs, hub: hub}
}
type runHandler struct {
http.Handler
runner *runner
hub *chatHub
}
// DetachSessions records the chats left live on their harness so the next
// start reattaches them. Call it when the server is stopping, before the
// handler's context is canceled (rooms are torn down after that) and before the
// store closes. It never interrupts a session.
func DetachSessions(handler http.Handler) {
h, ok := handler.(*runHandler)
if !ok || h.hub == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := h.hub.detachSessions(ctx); err != nil {
log.Printf("record live chat sessions: %v", err)
}
}
// WaitRuns drains active execution before the database is closed on shutdown.
+63 -19
View File
@@ -27,6 +27,23 @@ type AgentClient struct {
WorkStarted bool
}
// resumeConnection reopens the retained execution service a previous server
// process recorded for a session, refusing a different backend or directory.
func resumeConnection(reference, backend, directory string) (*opencode.Connection, error) {
if reference == "" {
return nil, fmt.Errorf("retained OpenCode session has no execution-service reference")
}
connection, err := opencode.FromReference(reference)
if err != nil {
return nil, err
}
if (backend != "" && connection.Backend != backend) || connection.Directory != directory {
connection.Close()
return nil, fmt.Errorf("retained OpenCode execution context mismatch")
}
return connection, nil
}
func (c OpenCodeClient) Interrupt(ctx context.Context, id string) error {
return c.interrupt(ctx, id, false)
}
@@ -80,7 +97,18 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
if c.Operations != nil {
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
}
if c.command == nil {
// A re-adopted session was admitted by a previous server process: attach
// to its retained service and wait for it; never create or prompt again.
resume := TakeResumeSession(ctx)
if resume != "" && c.command == nil {
connection, err := resumeConnection(c.Reference, c.Backend, c.Directory)
if err != nil {
return "", err
}
defer connection.Close()
c.connection = connection
c.Backend = connection.Backend
} else if c.command == nil {
connection, err := opencode.OpenStage(ctx, opencode.Executable(), c.Directory, c.Backend)
if err != nil {
// Connecting to the service failed before any session existed.
@@ -101,24 +129,31 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
if err != nil {
return "", err
}
data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules})
if err != nil {
if variant := model["variant"]; variant != "" {
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
if resume != "" {
if c.Backend != "v2" || !sessionIDPattern.MatchString(resume) {
return "", fmt.Errorf("retained OpenCode session %.128s cannot be re-adopted on backend %q", RedactEvidence(resume), c.Backend)
}
// No prompt or tool call was admitted: a request failure here is a
// reachable retry rather than a reason to end the run.
return "", harnessUnreachable(err)
c.SessionID = resume
} else {
data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules})
if err != nil {
if variant := model["variant"]; variant != "" {
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
}
// No prompt or tool call was admitted: a request failure here is a
// reachable retry rather than a reason to end the run.
return "", harnessUnreachable(err)
}
var created struct {
Data struct {
ID string `json:"id"`
} `json:"data"`
}
if json.Unmarshal(data, &created) != nil || !sessionIDPattern.MatchString(created.Data.ID) {
return "", fmt.Errorf("invalid agent session")
}
c.SessionID = created.Data.ID
}
var created struct {
Data struct {
ID string `json:"id"`
} `json:"data"`
}
if json.Unmarshal(data, &created) != nil || !sessionIDPattern.MatchString(created.Data.ID) {
return "", fmt.Errorf("invalid agent session")
}
c.SessionID = created.Data.ID
if c.connection != nil && c.Backend == "v1" {
c.Reference, err = c.connection.Reference()
if err != nil {
@@ -134,6 +169,12 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
defer func() {
watch.stopWatch(ctx, &err)
defer watch.finish(ctx, &err)
if err != nil && Detached(ctx) {
// The server is restarting: leave the admitted session running and
// its ownership recorded, so the next process re-adopts it.
err = fmt.Errorf("%w: retained OpenCode session %s left running", ErrDetached, c.SessionID)
return
}
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
defer cancel()
if err != nil {
@@ -181,8 +222,10 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
return "", err
}
c.WorkStarted = true
if _, err = c.call(ctx, "post", "/api/session/"+c.SessionID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil {
return "", err
if resume == "" {
if _, err = c.call(ctx, "post", "/api/session/"+c.SessionID+"/prompt", generationBody("text", prompt, config, c.Backend)); err != nil {
return "", err
}
}
if c.Backend == "v1" {
// Native V1 prompts block until completion; keep the existing settlement
@@ -193,6 +236,7 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
} else if err = c.waitForSessionIdle(ctx, c.SessionID); err != nil {
return "", err
}
var data []byte
data, text, err = c.readFinalAgentResult(ctx, c.SessionID)
if len(data) > 0 {
c.Transcript = string(data)
+61
View File
@@ -0,0 +1,61 @@
package stage
import (
"context"
"errors"
"sync"
)
// A Pipelines server restart must not stop harness work. The runner cancels a
// run's context with ErrServerShutdown as its cause when the server process is
// stopping (as opposed to an operator cancel, whose cause is context.Canceled).
// A stage client that sees this cause leaves its admitted session running: it
// neither interrupts it nor clears its recorded ownership, and returns an error
// wrapping ErrDetached. The next server process re-adopts the session through
// WithResumeSession and waits for it instead of prompting again.
// ErrServerShutdown is the cancellation cause of a run context whose server is
// stopping or restarting.
var ErrServerShutdown = errors.New("pipelines server is shutting down; harness sessions are left running")
// ErrDetached reports that a stage left its admitted harness session running
// for the next server process to re-adopt.
var ErrDetached = errors.New("stage session detached for server restart")
// Detached reports whether ctx was canceled because the server is stopping.
func Detached(ctx context.Context) bool {
return ctx.Err() != nil && errors.Is(context.Cause(ctx), ErrServerShutdown)
}
type resumeKey struct{}
type resumeHolder struct {
mu sync.Mutex
session string
}
// WithResumeSession marks the next model generation under ctx as a re-adoption
// of an already admitted session: the client attaches to sessionID, waits for it
// to settle and reads its final result, without creating a session or sending a
// prompt. The mark is consumed by the first generation, so a later retry or
// repair attempt starts fresh as usual.
func WithResumeSession(ctx context.Context, sessionID string) context.Context {
if sessionID == "" {
return ctx
}
return context.WithValue(ctx, resumeKey{}, &resumeHolder{session: sessionID})
}
// TakeResumeSession returns and consumes the session marked by
// WithResumeSession, or "" when the generation is a fresh one.
func TakeResumeSession(ctx context.Context) string {
holder, _ := ctx.Value(resumeKey{}).(*resumeHolder)
if holder == nil {
return ""
}
holder.mu.Lock()
defer holder.mu.Unlock()
id := holder.session
holder.session = ""
return id
}
+159
View File
@@ -0,0 +1,159 @@
package stage
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
)
// fakeV2Session answers the V2 calls a stage makes for one session, records
// every request, and reports the session running until finish is closed.
type fakeV2Session struct {
t *testing.T
id string
finish chan struct{}
mu sync.Mutex
calls []string
}
func (f *fakeV2Session) command(_ context.Context, _ string, args ...string) ([]byte, error) {
f.mu.Lock()
f.calls = append(f.calls, args[1]+" "+args[2])
f.mu.Unlock()
switch path := args[2]; {
case path == "/api/session":
return []byte(`{"data":{"id":"` + f.id + `"}}`), nil
case strings.HasSuffix(path, "/prompt"), strings.HasSuffix(path, "/interrupt"):
return []byte(`{}`), nil
case path == "/api/session/active":
select {
case <-f.finish:
return []byte(`{"data":{}}`), nil
default:
return []byte(`{"data":{"` + f.id + `":{"type":"running"}}}`), nil
}
case path == "/api/session/"+f.id+finalAssistantReadback:
return []byte(`{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n\nreattached reply\n"}]}]}`), nil
}
return nil, fmt.Errorf("unexpected request %v", args)
}
func (f *fakeV2Session) called(prefix string) bool {
f.mu.Lock()
defer f.mu.Unlock()
for _, call := range f.calls {
if strings.HasPrefix(call, prefix) {
return true
}
}
return false
}
// TestAgentDetachesOnServerShutdown: a server shutdown (not an operator cancel)
// leaves the admitted session running and owned: no interrupt, no release.
func TestAgentDetachesOnServerShutdown(t *testing.T) {
fake := &fakeV2Session{t: t, id: "ses_detach", finish: make(chan struct{})}
var released []string
client := &AgentClient{OnSession: func(id string, active bool) error {
if !active {
released = append(released, id)
}
return nil
}}
client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond
ctx, cancel := context.WithCancelCause(context.Background())
go func() {
for !fake.called("post /api/session/ses_detach/prompt") {
time.Sleep(time.Millisecond)
}
cancel(ErrServerShutdown)
}()
_, err := client.Generate(ctx, ModelConfig{}, "work")
if !errors.Is(err, ErrDetached) {
t.Fatalf("shutdown did not detach: %v", err)
}
if fake.called("post /api/session/ses_detach/interrupt") {
t.Fatal("a server shutdown interrupted the stage session")
}
if len(released) != 0 {
t.Fatalf("a detached session's ownership was released: %v", released)
}
}
// TestAgentOperatorCancelStillInterrupts keeps the explicit stop path.
func TestAgentOperatorCancelStillInterrupts(t *testing.T) {
fake := &fakeV2Session{t: t, id: "ses_cancel", finish: make(chan struct{})}
client := &AgentClient{}
client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond
ctx, cancel := context.WithCancel(context.Background())
go func() {
for !fake.called("post /api/session/ses_cancel/prompt") {
time.Sleep(time.Millisecond)
}
cancel()
// Interrupt confirmation waits for the session to settle.
for !fake.called("post /api/session/ses_cancel/interrupt") {
time.Sleep(time.Millisecond)
}
close(fake.finish)
}()
if _, err := client.Generate(ctx, ModelConfig{}, "work"); err == nil || errors.Is(err, ErrDetached) {
t.Fatalf("operator cancel did not fail the stage: %v", err)
}
if !fake.called("post /api/session/ses_cancel/interrupt") {
t.Fatal("operator cancel did not interrupt the session")
}
}
// TestAgentResumeAttachesWithoutPrompting: a re-adopted session is waited on
// and read back; it is never created or prompted again, and the resume mark is
// consumed so a retry starts fresh.
func TestAgentResumeAttachesWithoutPrompting(t *testing.T) {
fake := &fakeV2Session{t: t, id: "ses_adopted", finish: make(chan struct{})}
var owned []bool
client := &AgentClient{OnSession: func(id string, active bool) error {
if id != "ses_adopted" {
t.Errorf("ownership for %s", id)
}
owned = append(owned, active)
return nil
}}
client.Backend, client.Directory, client.command, client.sessionPollInterval = "v2", "/work", fake.command, time.Millisecond
ctx := WithResumeSession(context.Background(), "ses_adopted")
go func() {
time.Sleep(20 * time.Millisecond)
close(fake.finish)
}()
text, err := client.Generate(ctx, ModelConfig{}, "work")
if err != nil || !strings.Contains(text, "reattached reply") {
t.Fatalf("resume = %q, %v", text, err)
}
if fake.called("post /api/session ") || fake.called("post /api/session/ses_adopted/prompt") {
t.Fatalf("a re-adopted session was created or prompted again: %v", fake.calls)
}
if len(owned) != 2 || !owned[0] || owned[1] {
t.Fatalf("ownership was not re-asserted then released: %v", owned)
}
if TakeResumeSession(ctx) != "" {
t.Fatal("the resume mark was not consumed")
}
}
// TestPlainResumeReadsResultReachedWhileDown: a text-only session that ended
// during the downtime is folded in from its final assistant message.
func TestPlainResumeReadsResultReachedWhileDown(t *testing.T) {
fake := &fakeV2Session{t: t, id: "ses_plain", finish: make(chan struct{})}
close(fake.finish)
client := &OpenCodeClient{Backend: "v2", Directory: "/work", command: fake.command, sessionPollInterval: time.Millisecond}
text, err := client.Generate(WithResumeSession(context.Background(), "ses_plain"), ModelConfig{}, "prompt")
if err != nil || !strings.Contains(text, "reattached reply") {
t.Fatalf("resume = %q, %v", text, err)
}
if fake.called("post /api/session/ses_plain/prompt") || fake.called("post /api/session ") {
t.Fatalf("a re-adopted session was prompted again: %v", fake.calls)
}
}
+44 -19
View File
@@ -273,7 +273,18 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
if c.Operations != nil {
c.Operations.Provider, c.Operations.Model = config.Provider, config.Model
}
if c.command == nil {
// A re-adopted session was admitted by a previous server process (see
// WithResumeSession): wait for it and read its result, never prompt again.
resume := TakeResumeSession(ctx)
if resume != "" && c.command == nil {
connection, err := resumeConnection(c.Reference, c.Backend, c.Directory)
if err != nil {
return "", err
}
defer connection.Close()
c.connection = connection
c.Backend = connection.Backend
} else if c.command == nil {
connection, err := opencode.OpenStage(ctx, opencode.Executable(), c.Directory, c.Backend)
if err != nil {
// Connecting to the service failed before any session existed.
@@ -313,33 +324,45 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
if err != nil {
return "", err
}
created, err := c.call(ctx, "post", "/api/session", map[string]any{
"title": "Pipeline stage generation",
"location": map[string]string{"directory": directory},
"model": model,
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
})
if err != nil {
if variant := model["variant"]; variant != "" {
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
}
// No prompt or tool call was admitted: a request failure here is a
// reachable retry rather than a reason to end the run.
return "", harnessUnreachable(err)
}
var session struct {
Data struct {
ID string `json:"id"`
} `json:"data"`
}
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
return "", fmt.Errorf("OpenCode returned an invalid session ID")
if resume != "" {
if c.Backend != "v2" || !sessionIDPattern.MatchString(resume) {
return "", fmt.Errorf("retained OpenCode session %.128s cannot be re-adopted on backend %q", RedactEvidence(resume), c.Backend)
}
session.Data.ID = resume
} else {
created, err := c.call(ctx, "post", "/api/session", map[string]any{
"title": "Pipeline stage generation",
"location": map[string]string{"directory": directory},
"model": model,
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
})
if err != nil {
if variant := model["variant"]; variant != "" {
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
}
// No prompt or tool call was admitted: a request failure here is a
// reachable retry rather than a reason to end the run.
return "", harnessUnreachable(err)
}
if err := json.Unmarshal(created, &session); err != nil || !sessionIDPattern.MatchString(session.Data.ID) {
return "", fmt.Errorf("OpenCode returned an invalid session ID")
}
}
var watch *budgetWatch
promptStarted := false
defer func() {
watch.stopWatch(ctx, &err)
defer watch.finish(ctx, &err)
if err != nil && Detached(ctx) {
// The server is restarting: leave the session running and owned.
err = fmt.Errorf("%w: retained OpenCode session %s left running", ErrDetached, session.Data.ID)
return
}
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err != nil {
@@ -392,8 +415,10 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
// including unbudgeted text-only stages, so its exact native turns survive.
if c.Backend == "v2" {
promptStarted = true
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
return "", err
if resume == "" {
if _, err = c.call(ctx, "post", "/api/session/"+session.Data.ID+"/prompt", body); err != nil {
return "", err
}
}
if err = c.waitForSessionIdle(ctx, session.Data.ID); err != nil {
return "", err