Keep OpenCode/Codex chat and stage sessions running across server restarts

A stopping server no longer interrupts harness sessions. Runs are canceled
with a shutdown cause, so stage clients leave admitted sessions running and
owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait,
read back the result, continue the graph), restarts unadmitted visits,
continues runs between stages, and falls back to interrupted with a reason
when the harness is gone. Chats with live work are recorded on shutdown and
reattached on start; turns that ended during the downtime are folded in
from history. The server-spawned V2 chat serve is now a detached owned
service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
This commit is contained in:
2026-11-04 16:17:08 +00:00
parent 9351cd7d74
commit aa8eddd686
26 changed files with 2318 additions and 99 deletions
+7
View File
@@ -304,6 +304,13 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error {
return nil
}
// DetachSessions is the server-restart shutdown. Sessions, turns, questions,
// permissions and shells live in the OpenCode service, not in this adapter, so
// there is nothing to stop: the connection's owner decides whether the service
// itself outlives the server (an attached endpoint and the detached owned chat
// service do; a V1 serve child does not).
func (c *Chat) DetachSessions() {}
// Messages reads one session's stored turns. On the owned serve backend it
// reuses Connection.Messages, including its read-only OutputFormat readback
// fallback, instead of duplicating that path. It maps the verified assistant
+64
View File
@@ -0,0 +1,64 @@
package opencode
import (
"context"
"errors"
"strings"
"syscall"
"testing"
)
// The server-spawned chat service is detached and registered: a second server
// process reuses the same running service (sessions survive the restart), and
// it is separate from the stage service.
func TestOwnedChatServiceIsReusedAcrossServerProcesses(t *testing.T) {
executable := fakeOwnedStageEnv(t)
t.Setenv(ChatConfigEnvKey, "")
t.Cleanup(func() {
path, _ := ChatServiceRegistrationPath()
if state, err := readOwnedStageState(path); err == nil {
_ = syscall.Kill(state.PID, syscall.SIGKILL)
}
})
first, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir())
if err != nil {
t.Fatal(err)
}
path, _ := ChatServiceRegistrationPath()
stagePath, _ := StageServiceRegistrationPath()
if first.credentialFile != path || path == stagePath || !strings.HasPrefix(first.URL, "http://127.0.0.1:") {
t.Fatalf("chat connection = %+v", first)
}
pid := ownedPID(t, first)
first.Close() // the previous server process exits
second, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir())
if err != nil || second.URL != first.URL || ownedPID(t, second) != pid {
t.Fatalf("chat service was not reused by the next server: %v", err)
}
}
// ReferenceAlive tells a live retained service from one whose process is gone,
// without reviving it.
func TestReferenceAliveDoesNotRevive(t *testing.T) {
executable := fakeOwnedStageEnv(t)
c, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
if err != nil {
t.Fatal(err)
}
reference, err := c.Reference()
if err != nil {
t.Fatal(err)
}
if err := ReferenceAlive(context.Background(), reference); err != nil {
t.Fatalf("live service reported gone: %v", err)
}
pid := ownedPID(t, c)
_ = syscall.Kill(pid, syscall.SIGKILL)
waitGone(t, pid)
if err := ReferenceAlive(context.Background(), reference); !errors.Is(err, ErrServiceNotRunning) {
t.Fatalf("gone service not reported: %v", err)
}
if _, healthy := c.ownedInfo(context.Background()); healthy {
t.Fatal("ReferenceAlive revived the service")
}
}
+50 -7
View File
@@ -8,6 +8,7 @@ import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -90,23 +91,65 @@ func (c *Connection) Reference() (string, error) {
return f.Name(), nil
}
func FromReference(path string) (*Connection, error) {
type connectionReference struct{ Backend, URL, Password, Directory, CredentialFile string }
func readReference(path string) (connectionReference, error) {
var reference connectionReference
info, err := os.Lstat(path)
if err != nil {
return nil, err
return reference, err
}
if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || info.Size() > MaxRequest {
return nil, fmt.Errorf("invalid private OpenCode connection reference")
return reference, fmt.Errorf("invalid private OpenCode connection reference")
}
if stat, ok := info.Sys().(*syscall.Stat_t); !ok || stat.Uid != uint32(os.Getuid()) {
return nil, fmt.Errorf("OpenCode reference belongs to another user")
return reference, fmt.Errorf("OpenCode reference belongs to another user")
}
data, err := os.ReadFile(path)
if err != nil {
return nil, err
return reference, err
}
var reference struct{ Backend, URL, Password, Directory, CredentialFile string }
if err = json.Unmarshal(data, &reference); err != nil {
err = json.Unmarshal(data, &reference)
return reference, err
}
// ErrServiceNotRunning reports that a retained OpenCode service no longer
// answers: its process is gone, and with it any session turn it was running.
var ErrServiceNotRunning = errors.New("the OpenCode service is not running")
// ReferenceAlive reports whether the execution service a retained reference
// names is running right now, without starting or reviving it. Restart recovery
// uses it to tell a session that kept running across a Pipelines restart from
// one whose harness process is gone.
func ReferenceAlive(ctx context.Context, path string) error {
reference, err := readReference(path)
if err != nil {
return fmt.Errorf("retained OpenCode reference unreadable: %w", err)
}
if reference.Backend != "v2" {
return fmt.Errorf("the V1 OpenCode stage service belonged to the stopped server process")
}
if !filepath.IsAbs(reference.CredentialFile) {
return fmt.Errorf("invalid V2 service credential reference")
}
c, err := serviceConnection(reference.CredentialFile, reference.Directory)
if err != nil {
return err
}
if c.URL != reference.URL {
return fmt.Errorf("retained OpenCode service endpoint changed")
}
probe, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
if err := c.probeHealth(probe, "v2"); err != nil {
return fmt.Errorf("%w at %s", ErrServiceNotRunning, c.URL)
}
return nil
}
func FromReference(path string) (*Connection, error) {
reference, err := readReference(path)
if err != nil {
return nil, err
}
if reference.Backend == "v2" {
+60 -14
View File
@@ -54,14 +54,56 @@ type ownedStageState struct {
Identity string `json:"identity"`
}
// StageServiceRegistrationPath is the private registration of the owned stage
// service, beside the private connection references.
func StageServiceRegistrationPath() (string, error) {
// ownedService names one Pipelines-owned detached `opencode serve`: the stage
// service, or the chat service a server without an attached endpoint uses.
type ownedService struct {
name string // for logs: "stage" or "chat"
file string // registration file name, without .json
}
var (
stageOwnedService = ownedService{name: "stage", file: "opencode-stage-service"}
chatOwnedService = ownedService{name: "chat", file: "opencode-chat-service"}
)
func (s ownedService) registrationPath() (string, error) {
cache, err := os.UserCacheDir()
if err != nil {
return "", err
}
return filepath.Join(cache, "pipelines", "private", "opencode-stage-service.json"), nil
return filepath.Join(cache, "pipelines", "private", s.file+".json"), nil
}
// StageServiceRegistrationPath is the private registration of the owned stage
// service, beside the private connection references.
func StageServiceRegistrationPath() (string, error) {
return stageOwnedService.registrationPath()
}
// ChatServiceRegistrationPath is the private registration of the owned chat
// service (see EnsureOwnedChatService).
func ChatServiceRegistrationPath() (string, error) {
return chatOwnedService.registrationPath()
}
// EnsureOwnedChatService returns a connection to the detached Pipelines-owned
// V2 chat service, starting or reviving it when needed. Unlike a serve child of
// the server, it keeps running when the server stops, so chat turns, questions
// and background shells survive a restart; the next server reuses the same
// endpoint and password from the private registration. Chat-only limits
// (PIPELINES_OPENCODE_CHAT_CONFIG) apply to it alone.
func EnsureOwnedChatService(ctx context.Context, executable, directory string) (*Connection, error) {
overrides, err := chatConfigOverrides()
if err != nil {
return nil, err
}
content := ""
if overrides != nil {
if content, err = configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), overrides); err != nil {
return nil, err
}
}
return ensureOwnedService(ctx, chatOwnedService, content, executable, directory)
}
func isOwnedStageRegistration(path string) bool {
@@ -81,7 +123,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
if err != nil {
return nil, err
}
path, err := StageServiceRegistrationPath()
return ensureOwnedService(ctx, stageOwnedService, content, executable, directory)
}
func ensureOwnedService(ctx context.Context, service ownedService, content, executable, directory string) (*Connection, error) {
path, err := service.registrationPath()
if err != nil {
return nil, err
}
@@ -92,7 +138,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
defer ownedStageMu.Unlock()
unlock, err := lockFile(path + ".lock")
if err != nil {
return nil, fmt.Errorf("OpenCode stage service lock: %w", err)
return nil, fmt.Errorf("OpenCode %s service lock: %w", service.name, err)
}
defer unlock()
@@ -106,11 +152,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
case healthy && state.Identity == identity:
return ownedStageConnection(path, executable, directory)
case healthy && current.ownedIdle(ctx):
log.Printf("OpenCode stage service configuration changed; restarting the idle service")
log.Printf("OpenCode %s service configuration changed; restarting the idle service", service.name)
stopOwnedPID(pid)
case healthy:
// Never interrupt active stage work to apply a new limit.
log.Printf("OpenCode stage service configuration changed; keeping the busy service until it is idle")
log.Printf("OpenCode %s service configuration changed; keeping the busy service until it is idle", service.name)
return ownedStageConnection(path, executable, directory)
}
if u, err := url.Parse(state.URL); err == nil {
@@ -135,7 +181,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
case <-time.After(time.Duration(attempt) * time.Second):
}
}
started, err := startOwnedStage(ctx, executable, port, state.Password, content)
started, err := startOwnedStage(ctx, service, executable, port, state.Password, content)
if err == nil {
started.Identity = identity
if err := writeOwnedStageState(path, started); err != nil {
@@ -197,7 +243,7 @@ func ownedStageIdentity(executable, content string) string {
// startOwnedStage starts one detached serve process and waits until it answers
// the authenticated V2 info contract.
func startOwnedStage(ctx context.Context, executable string, port int, password, content string) (ownedStageState, error) {
func startOwnedStage(ctx context.Context, service ownedService, executable string, port int, password, content string) (ownedStageState, error) {
if port == 0 {
free, err := freePort()
if err != nil {
@@ -214,7 +260,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
// Its own session: a stopping Pipelines service (and its process group)
// does not take active stage work with it.
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
logPath, _ := StageServiceRegistrationPath()
logPath, _ := service.registrationPath()
logPath = strings.TrimSuffix(logPath, ".json") + ".log"
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
@@ -224,7 +270,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
cmd.Stdin = nil
if err := cmd.Start(); err != nil {
logFile.Close()
return ownedStageState{}, fmt.Errorf("OpenCode stage service could not start: %w", err)
return ownedStageState{}, fmt.Errorf("OpenCode %s service could not start: %w", service.name, err)
}
logFile.Close()
exited := make(chan struct{})
@@ -241,13 +287,13 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
}
select {
case <-exited:
return ownedStageState{}, fmt.Errorf("OpenCode stage service exited before becoming ready (see %s)", logPath)
return ownedStageState{}, fmt.Errorf("OpenCode %s service exited before becoming ready (see %s)", service.name, logPath)
case <-startup.Done():
_ = cmd.Process.Kill()
if ctx.Err() != nil {
return ownedStageState{}, ctx.Err()
}
return ownedStageState{}, fmt.Errorf("OpenCode stage service did not become ready within %s (see %s)", ownedStageStartTimeout, logPath)
return ownedStageState{}, fmt.Errorf("OpenCode %s service did not become ready within %s (see %s)", service.name, ownedStageStartTimeout, logPath)
case <-time.After(150 * time.Millisecond):
}
}