Keep OpenCode/Codex chat and stage sessions running across server restarts
A stopping server no longer interrupts harness sessions. Runs are canceled with a shutdown cause, so stage clients leave admitted sessions running and owned; start-up recovery re-adopts live OpenCode V2 stage sessions (wait, read back the result, continue the graph), restarts unadmitted visits, continues runs between stages, and falls back to interrupted with a reason when the harness is gone. Chats with live work are recorded on shutdown and reattached on start; turns that ended during the downtime are folded in from history. The server-spawned V2 chat serve is now a detached owned service. PIPELINES_SHUTDOWN_SESSIONS=stop restores the old behaviour.
This commit is contained in:
@@ -304,6 +304,13 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DetachSessions is the server-restart shutdown. Sessions, turns, questions,
|
||||
// permissions and shells live in the OpenCode service, not in this adapter, so
|
||||
// there is nothing to stop: the connection's owner decides whether the service
|
||||
// itself outlives the server (an attached endpoint and the detached owned chat
|
||||
// service do; a V1 serve child does not).
|
||||
func (c *Chat) DetachSessions() {}
|
||||
|
||||
// Messages reads one session's stored turns. On the owned serve backend it
|
||||
// reuses Connection.Messages, including its read-only OutputFormat readback
|
||||
// fallback, instead of duplicating that path. It maps the verified assistant
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The server-spawned chat service is detached and registered: a second server
|
||||
// process reuses the same running service (sessions survive the restart), and
|
||||
// it is separate from the stage service.
|
||||
func TestOwnedChatServiceIsReusedAcrossServerProcesses(t *testing.T) {
|
||||
executable := fakeOwnedStageEnv(t)
|
||||
t.Setenv(ChatConfigEnvKey, "")
|
||||
t.Cleanup(func() {
|
||||
path, _ := ChatServiceRegistrationPath()
|
||||
if state, err := readOwnedStageState(path); err == nil {
|
||||
_ = syscall.Kill(state.PID, syscall.SIGKILL)
|
||||
}
|
||||
})
|
||||
first, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path, _ := ChatServiceRegistrationPath()
|
||||
stagePath, _ := StageServiceRegistrationPath()
|
||||
if first.credentialFile != path || path == stagePath || !strings.HasPrefix(first.URL, "http://127.0.0.1:") {
|
||||
t.Fatalf("chat connection = %+v", first)
|
||||
}
|
||||
pid := ownedPID(t, first)
|
||||
first.Close() // the previous server process exits
|
||||
second, err := EnsureOwnedChatService(context.Background(), executable, t.TempDir())
|
||||
if err != nil || second.URL != first.URL || ownedPID(t, second) != pid {
|
||||
t.Fatalf("chat service was not reused by the next server: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ReferenceAlive tells a live retained service from one whose process is gone,
|
||||
// without reviving it.
|
||||
func TestReferenceAliveDoesNotRevive(t *testing.T) {
|
||||
executable := fakeOwnedStageEnv(t)
|
||||
c, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reference, err := c.Reference()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ReferenceAlive(context.Background(), reference); err != nil {
|
||||
t.Fatalf("live service reported gone: %v", err)
|
||||
}
|
||||
pid := ownedPID(t, c)
|
||||
_ = syscall.Kill(pid, syscall.SIGKILL)
|
||||
waitGone(t, pid)
|
||||
if err := ReferenceAlive(context.Background(), reference); !errors.Is(err, ErrServiceNotRunning) {
|
||||
t.Fatalf("gone service not reported: %v", err)
|
||||
}
|
||||
if _, healthy := c.ownedInfo(context.Background()); healthy {
|
||||
t.Fatal("ReferenceAlive revived the service")
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -90,23 +91,65 @@ func (c *Connection) Reference() (string, error) {
|
||||
return f.Name(), nil
|
||||
}
|
||||
|
||||
func FromReference(path string) (*Connection, error) {
|
||||
type connectionReference struct{ Backend, URL, Password, Directory, CredentialFile string }
|
||||
|
||||
func readReference(path string) (connectionReference, error) {
|
||||
var reference connectionReference
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return reference, err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || info.Size() > MaxRequest {
|
||||
return nil, fmt.Errorf("invalid private OpenCode connection reference")
|
||||
return reference, fmt.Errorf("invalid private OpenCode connection reference")
|
||||
}
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); !ok || stat.Uid != uint32(os.Getuid()) {
|
||||
return nil, fmt.Errorf("OpenCode reference belongs to another user")
|
||||
return reference, fmt.Errorf("OpenCode reference belongs to another user")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return reference, err
|
||||
}
|
||||
var reference struct{ Backend, URL, Password, Directory, CredentialFile string }
|
||||
if err = json.Unmarshal(data, &reference); err != nil {
|
||||
err = json.Unmarshal(data, &reference)
|
||||
return reference, err
|
||||
}
|
||||
|
||||
// ErrServiceNotRunning reports that a retained OpenCode service no longer
|
||||
// answers: its process is gone, and with it any session turn it was running.
|
||||
var ErrServiceNotRunning = errors.New("the OpenCode service is not running")
|
||||
|
||||
// ReferenceAlive reports whether the execution service a retained reference
|
||||
// names is running right now, without starting or reviving it. Restart recovery
|
||||
// uses it to tell a session that kept running across a Pipelines restart from
|
||||
// one whose harness process is gone.
|
||||
func ReferenceAlive(ctx context.Context, path string) error {
|
||||
reference, err := readReference(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("retained OpenCode reference unreadable: %w", err)
|
||||
}
|
||||
if reference.Backend != "v2" {
|
||||
return fmt.Errorf("the V1 OpenCode stage service belonged to the stopped server process")
|
||||
}
|
||||
if !filepath.IsAbs(reference.CredentialFile) {
|
||||
return fmt.Errorf("invalid V2 service credential reference")
|
||||
}
|
||||
c, err := serviceConnection(reference.CredentialFile, reference.Directory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.URL != reference.URL {
|
||||
return fmt.Errorf("retained OpenCode service endpoint changed")
|
||||
}
|
||||
probe, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
if err := c.probeHealth(probe, "v2"); err != nil {
|
||||
return fmt.Errorf("%w at %s", ErrServiceNotRunning, c.URL)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func FromReference(path string) (*Connection, error) {
|
||||
reference, err := readReference(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reference.Backend == "v2" {
|
||||
|
||||
@@ -54,14 +54,56 @@ type ownedStageState struct {
|
||||
Identity string `json:"identity"`
|
||||
}
|
||||
|
||||
// StageServiceRegistrationPath is the private registration of the owned stage
|
||||
// service, beside the private connection references.
|
||||
func StageServiceRegistrationPath() (string, error) {
|
||||
// ownedService names one Pipelines-owned detached `opencode serve`: the stage
|
||||
// service, or the chat service a server without an attached endpoint uses.
|
||||
type ownedService struct {
|
||||
name string // for logs: "stage" or "chat"
|
||||
file string // registration file name, without .json
|
||||
}
|
||||
|
||||
var (
|
||||
stageOwnedService = ownedService{name: "stage", file: "opencode-stage-service"}
|
||||
chatOwnedService = ownedService{name: "chat", file: "opencode-chat-service"}
|
||||
)
|
||||
|
||||
func (s ownedService) registrationPath() (string, error) {
|
||||
cache, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(cache, "pipelines", "private", "opencode-stage-service.json"), nil
|
||||
return filepath.Join(cache, "pipelines", "private", s.file+".json"), nil
|
||||
}
|
||||
|
||||
// StageServiceRegistrationPath is the private registration of the owned stage
|
||||
// service, beside the private connection references.
|
||||
func StageServiceRegistrationPath() (string, error) {
|
||||
return stageOwnedService.registrationPath()
|
||||
}
|
||||
|
||||
// ChatServiceRegistrationPath is the private registration of the owned chat
|
||||
// service (see EnsureOwnedChatService).
|
||||
func ChatServiceRegistrationPath() (string, error) {
|
||||
return chatOwnedService.registrationPath()
|
||||
}
|
||||
|
||||
// EnsureOwnedChatService returns a connection to the detached Pipelines-owned
|
||||
// V2 chat service, starting or reviving it when needed. Unlike a serve child of
|
||||
// the server, it keeps running when the server stops, so chat turns, questions
|
||||
// and background shells survive a restart; the next server reuses the same
|
||||
// endpoint and password from the private registration. Chat-only limits
|
||||
// (PIPELINES_OPENCODE_CHAT_CONFIG) apply to it alone.
|
||||
func EnsureOwnedChatService(ctx context.Context, executable, directory string) (*Connection, error) {
|
||||
overrides, err := chatConfigOverrides()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content := ""
|
||||
if overrides != nil {
|
||||
if content, err = configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), overrides); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return ensureOwnedService(ctx, chatOwnedService, content, executable, directory)
|
||||
}
|
||||
|
||||
func isOwnedStageRegistration(path string) bool {
|
||||
@@ -81,7 +123,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path, err := StageServiceRegistrationPath()
|
||||
return ensureOwnedService(ctx, stageOwnedService, content, executable, directory)
|
||||
}
|
||||
|
||||
func ensureOwnedService(ctx context.Context, service ownedService, content, executable, directory string) (*Connection, error) {
|
||||
path, err := service.registrationPath()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -92,7 +138,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
|
||||
defer ownedStageMu.Unlock()
|
||||
unlock, err := lockFile(path + ".lock")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("OpenCode stage service lock: %w", err)
|
||||
return nil, fmt.Errorf("OpenCode %s service lock: %w", service.name, err)
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
@@ -106,11 +152,11 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
|
||||
case healthy && state.Identity == identity:
|
||||
return ownedStageConnection(path, executable, directory)
|
||||
case healthy && current.ownedIdle(ctx):
|
||||
log.Printf("OpenCode stage service configuration changed; restarting the idle service")
|
||||
log.Printf("OpenCode %s service configuration changed; restarting the idle service", service.name)
|
||||
stopOwnedPID(pid)
|
||||
case healthy:
|
||||
// Never interrupt active stage work to apply a new limit.
|
||||
log.Printf("OpenCode stage service configuration changed; keeping the busy service until it is idle")
|
||||
log.Printf("OpenCode %s service configuration changed; keeping the busy service until it is idle", service.name)
|
||||
return ownedStageConnection(path, executable, directory)
|
||||
}
|
||||
if u, err := url.Parse(state.URL); err == nil {
|
||||
@@ -135,7 +181,7 @@ func ensureOwnedStageService(ctx context.Context, executable, directory string)
|
||||
case <-time.After(time.Duration(attempt) * time.Second):
|
||||
}
|
||||
}
|
||||
started, err := startOwnedStage(ctx, executable, port, state.Password, content)
|
||||
started, err := startOwnedStage(ctx, service, executable, port, state.Password, content)
|
||||
if err == nil {
|
||||
started.Identity = identity
|
||||
if err := writeOwnedStageState(path, started); err != nil {
|
||||
@@ -197,7 +243,7 @@ func ownedStageIdentity(executable, content string) string {
|
||||
|
||||
// startOwnedStage starts one detached serve process and waits until it answers
|
||||
// the authenticated V2 info contract.
|
||||
func startOwnedStage(ctx context.Context, executable string, port int, password, content string) (ownedStageState, error) {
|
||||
func startOwnedStage(ctx context.Context, service ownedService, executable string, port int, password, content string) (ownedStageState, error) {
|
||||
if port == 0 {
|
||||
free, err := freePort()
|
||||
if err != nil {
|
||||
@@ -214,7 +260,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
|
||||
// Its own session: a stopping Pipelines service (and its process group)
|
||||
// does not take active stage work with it.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
logPath, _ := StageServiceRegistrationPath()
|
||||
logPath, _ := service.registrationPath()
|
||||
logPath = strings.TrimSuffix(logPath, ".json") + ".log"
|
||||
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
@@ -224,7 +270,7 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
|
||||
cmd.Stdin = nil
|
||||
if err := cmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode stage service could not start: %w", err)
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode %s service could not start: %w", service.name, err)
|
||||
}
|
||||
logFile.Close()
|
||||
exited := make(chan struct{})
|
||||
@@ -241,13 +287,13 @@ func startOwnedStage(ctx context.Context, executable string, port int, password,
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode stage service exited before becoming ready (see %s)", logPath)
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode %s service exited before becoming ready (see %s)", service.name, logPath)
|
||||
case <-startup.Done():
|
||||
_ = cmd.Process.Kill()
|
||||
if ctx.Err() != nil {
|
||||
return ownedStageState{}, ctx.Err()
|
||||
}
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode stage service did not become ready within %s (see %s)", ownedStageStartTimeout, logPath)
|
||||
return ownedStageState{}, fmt.Errorf("OpenCode %s service did not become ready within %s (see %s)", service.name, ownedStageStartTimeout, logPath)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user