Keep OpenCode/Codex chat and stage sessions running across server restarts

A stopping server no longer interrupts harness sessions. Runs are canceled
with a shutdown cause, so stage clients leave admitted sessions running and
owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait,
read back the result, continue the graph), restarts unadmitted visits,
continues runs between stages, and falls back to interrupted with a reason
when the harness is gone. Chats with live work are recorded on shutdown and
reattached on start; turns that ended during the downtime are folded in
from history. The server-spawned V2 chat serve is now a detached owned
service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
This commit is contained in:
2026-11-04 16:17:08 +00:00
parent 9351cd7d74
commit aa8eddd686
26 changed files with 2318 additions and 99 deletions
+61 -5
View File
@@ -49,8 +49,18 @@ func main() {
if err := store.AcquireServerLock(); err != nil {
log.Fatal(err)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
signals, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
// The server's own context ends after a signal, once the chats left live on
// their harness have been recorded (see the shutdown goroutine below).
ctx, cancelServer := context.WithCancel(context.Background())
defer cancelServer()
// A restart or stop leaves harness sessions running and the next start
// reattaches them; PIPELINES_SHUTDOWN_SESSIONS=stop restores stopping them.
detach, err := shutdownDetaches(os.Getenv(shutdownSessionsEnvKey))
if err != nil {
log.Fatal(err)
}
// A brand-new database starts with the bundled examples; an existing or
// already-seeded one is never changed, even if its pipelines were deleted.
if seeded, err := server.SeedExamples(ctx, store); err != nil {
@@ -62,7 +72,7 @@ func main() {
// Each new run gets a bounded one-line title from the configured default
// harness; the summariser falls back to the prompt's first line, so title
// generation never blocks or fails run creation.
options := server.Options{AllowAuthoring: *allowAuthoring, ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)}
options := server.Options{AllowAuthoring: *allowAuthoring, ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv), DetachSessions: detach}
recorded := ""
if value, err := store.OpenCodeEndpoint(ctx); err != nil {
log.Printf("OpenCode endpoint setting unreadable: %v", err)
@@ -117,7 +127,7 @@ func main() {
}
}
if options.Chat == nil {
if conn, err := opencode.OpenChat(ctx, opencode.Executable(), ""); err != nil {
if conn, err := openChat(ctx, detach); err != nil {
log.Printf("OpenCode chat unavailable: %v", err)
} else if conn.URL == "" {
conn.Close()
@@ -141,7 +151,11 @@ func main() {
chats := server.NewChatManager(ctx, options.Chat, store)
if len(chats.Backends()) > 0 {
options.Chat = chats
defer chats.Close()
if detach {
defer chats.DetachSessions()
} else {
defer chats.Close()
}
}
srv := &http.Server{Addr: *addr, Handler: server.HandlerWithOptions(ctx, options, store), ReadHeaderTimeout: 5 * time.Second}
// Deferred before WaitRuns so it runs after the runs settle: the owned stage
@@ -149,7 +163,16 @@ func main() {
defer opencode.StopIdleStageService()
defer server.WaitRuns(srv.Handler)
go func() {
<-ctx.Done()
select {
case <-signals.Done():
case <-ctx.Done():
}
if detach {
// Record the live chats while their rooms still exist.
server.DetachSessions(srv.Handler)
log.Printf("shutdown: harness sessions are left running; the next start reattaches them")
}
cancelServer()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
@@ -172,3 +195,36 @@ func main() {
log.Fatal(err)
}
}
// shutdownSessionsEnvKey selects what a stopping server does with harness
// sessions: "keep" (the default) leaves them running for the next start to
// reattach; "stop" interrupts runs and closes chat clients as before.
const shutdownSessionsEnvKey = "PIPELINES_SHUTDOWN_SESSIONS"
func shutdownDetaches(value string) (bool, error) {
switch value {
case "", "keep":
return true, nil
case "stop":
return false, nil
}
return false, errors.New(shutdownSessionsEnvKey + " must be keep or stop")
}
// openChat opens the OpenCode service chat brokers from when no endpoint is
// attached. When sessions are kept across restarts, a V2 installation uses the
// detached Pipelines-owned chat service, which outlives this process; otherwise
// (and for V1) it is a serve child that ends with the server.
func openChat(ctx context.Context, detach bool) (*opencode.Connection, error) {
executable := opencode.Executable()
if detach {
if backend, err := opencode.Detect(ctx, executable); err == nil && backend == "v2" {
directory, err := os.Getwd()
if err != nil {
return nil, err
}
return opencode.EnsureOwnedChatService(ctx, executable, directory)
}
}
return opencode.OpenChat(ctx, executable, "")
}