Preserve independent background sessions through owner shutdown and idle cleanup

This commit is contained in:
2026-11-07 16:23:20 +00:00
parent 4ead2abdbf
commit b15597662f
9 changed files with 360 additions and 51 deletions
+14 -10
View File
@@ -96,8 +96,8 @@ Ordinary notifications are not spooled; the existing thread readback repairs gap
State is bounded to 128 callbacks / 8 MiB and 256 in-flight client calls, with a
16 MiB frame limit. Exceeding pending bounds closes the bridge without granting,
declining or otherwise auto-answering an interaction. A detached bridge expires
after 20 minutes with no pending interactions, or after 24 hours even if pending.
declining or otherwise auto-answering an interaction. A detached bridge keeps its original upstream connection indefinitely by default.
Only explicitly supplied positive `-idle`/`-max-detached` limits enable expiry.
An upstream App Server restart, a bridge crash, or expiry loses its connection-
bound callbacks: no disk JSON can recreate them on a new connection. The initial
deployment cannot recover requests already owned by the old direct connection.
@@ -161,16 +161,20 @@ The host stops the CLI (EOF, then SIGINT, then SIGKILL to its process group) and
removes its files only when one of these happens:
- The operator closes the chat (`chat_close`).
- The server's idle release or live-process cap fires.
- A genuinely idle unhosted CLI is released. For hosted sessions, server cache
eviction only detaches; the host owns authoritative background-task liveness.
- The CLI exits; the server then gets its stderr.
- With no server attached, the CLI has been idle for the same 20 minutes, or
no server has attached for 24 hours.
- With no server attached, the CLI has been genuinely idle for 20 minutes,
with no foreground turn, pending interaction, or native background task.
There is no default hard detached lifetime limit.
If the host itself dies, the server first tries to reconnect once. If that fails,
it stops the orphaned CLI's process group (only when that pid's command line
still names the session) and removes the host files. A running turn ends with
the notice "the Claude Code host stopped unexpectedly; the next prompt resumes
this session with --resume". The next prompt then resumes the session.
If the control connection is lost, the server first tries to reconnect. A verified
live host is preserved when its socket is temporarily unavailable, and another
CLI is not started alongside it. Dead host recovery verifies process identity
before stopping an orphan. On attachment, the host's authoritative task snapshot
restores running task identities even after older spool records were trimmed.
Legacy hosts without that snapshot remain attached during cache eviction, so
their older detached idle timers cannot mistake background work for an idle CLI.
Limits:
+49
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
@@ -150,3 +151,51 @@ func TestProcessExitEndsBackgroundTasks(t *testing.T) {
t.Fatal("no shell exit was published")
}
}
func TestObserverReleaseDoesNotForgetOrReapRunningNativeTasks(t *testing.T) {
chat, s := backgroundSession(t)
defer chat.Close()
s.handleLine(nil, toolUse(t, "toolu_bg", "Bash", map[string]any{"command": "quiet background work", "run_in_background": true}))
s.handleLine(nil, toolResult(t, "toolu_bg", "Command running in background with ID: native-bg"))
s.handleLine(nil, toolUse(t, "toolu_done", "Bash", map[string]any{"command": "completed foreground"}))
s.handleLine(nil, toolResult(t, "toolu_done", "done"))
proc := newProcess(s, "")
proc.markReady()
s.mu.Lock()
s.proc = proc
s.running = false
s.lastUsed = time.Now().Add(-time.Hour)
s.mu.Unlock()
chat.ReleaseShells(s.id)
if shells := running(t, chat); len(shells) != 1 || shells[0].ID != shellID("toolu_bg") {
t.Fatalf("observer expiry lost task identity: %+v", shells)
}
chat.reapIdleAt(time.Now())
s.mu.Lock()
same := s.proc == proc
s.proc = nil
s.mu.Unlock()
if !same {
t.Fatal("idle cache eviction stopped active background work")
}
}
func TestLegacyHostRemainsAttachedDuringIdleCacheEviction(t *testing.T) {
chat, s := backgroundSession(t)
defer chat.Close()
proc := newProcess(s, "")
proc.markReady()
proc.link = &hostLink{} // No authoritative task snapshot capability.
s.mu.Lock()
s.proc = proc
s.lastUsed = time.Now().Add(-time.Hour)
s.mu.Unlock()
chat.reapIdleAt(time.Now())
s.mu.Lock()
same := s.proc == proc
s.proc = nil
s.mu.Unlock()
if !same {
t.Fatal("legacy host was detached despite unknown background liveness")
}
}
+87 -31
View File
@@ -161,13 +161,27 @@ func (c *Chat) Close() {
}
}
// ReleaseShells drops a session's retained shell snapshot after the last
// observer leaves. A later attach re-reads the authoritative transcript.
// ReleaseShells releases completed display snapshots, never liveness for
// running tasks. A room/observer expiry is not permission to stop native work.
func (c *Chat) ReleaseShells(id string) {
if s := c.session(id); s != nil {
s.mu.Lock()
s.shells = map[string]*shellSnapshot{}
s.shellOrder = nil
kept := make(map[string]*shellSnapshot)
ids := map[string]bool{}
for key, task := range s.shells {
if task.shell.Status == protocol.ChatShellRunning {
kept[key] = task
ids[task.shell.ID] = true
}
}
var order []string
for _, id := range s.shellOrder {
if ids[id] {
order = append(order, id)
}
}
s.shells, s.shellOrder = kept, order
s.mu.Unlock()
}
}
@@ -250,37 +264,46 @@ func (c *Chat) reapIdle() {
return
case <-ticker.C:
}
now := time.Now()
type candidate struct {
s *session
used time.Time
alive bool
c.reapIdleAt(time.Now())
}
}
func (c *Chat) reapIdleAt(now time.Time) {
type candidate struct {
s *session
used time.Time
}
var live []candidate
c.mu.Lock()
for _, s := range c.sessions {
s.mu.Lock()
alive := s.proc != nil
used := s.lastUsed
s.mu.Unlock()
if alive {
live = append(live, candidate{s: s, used: used})
}
var live []candidate
c.mu.Lock()
for _, s := range c.sessions {
s.mu.Lock()
alive := s.proc != nil
used := s.lastUsed
s.mu.Unlock()
if alive {
live = append(live, candidate{s: s, used: used})
}
}
c.mu.Unlock()
sort.Slice(live, func(i, j int) bool { return live[i].used.Before(live[j].used) })
for _, entry := range live {
entry.s.mu.Lock()
excess := len(live) > maxLiveProcesses && !entry.s.running
idle := !entry.s.running && len(entry.s.subs) == 0 && now.Sub(entry.used) > idleProcessExpiry
entry.s.mu.Unlock()
if excess || idle {
entry.s.closeProcess()
// Recompute the live count on the next tick after a close.
live = live[1:]
}
c.mu.Unlock()
sort.Slice(live, func(i, j int) bool { return live[i].used.Before(live[j].used) })
for _, entry := range live {
entry.s.mu.Lock()
busy := entry.s.running
for _, task := range entry.s.shells {
if task.shell.Status == protocol.ChatShellRunning {
busy = true
break
}
}
excess := len(live) > maxLiveProcesses && !busy
idle := !busy && len(entry.s.subs) == 0 && now.Sub(entry.used) > idleProcessExpiry
entry.s.mu.Unlock()
if excess || idle {
entry.s.releaseIdleProcess()
// Recompute the live count on the next tick after a close.
live = live[1:]
break
}
}
}
@@ -443,6 +466,39 @@ func (s *session) unsubscribe(ch chan protocol.ChatEventFrame) {
s.mu.Unlock()
}
// The cache reaper may relinquish an idle control connection, not stop a
// hosted session. Only the host knows authoritative tasks beyond spool replay;
// an explicit operator close remains the hard-close path below.
func (s *session) releaseIdleProcess() {
s.mu.Lock()
if s.running || len(s.subs) > 0 {
s.mu.Unlock()
return
}
for _, task := range s.shells {
if task.shell.Status == protocol.ChatShellRunning {
s.mu.Unlock()
return
}
}
p := s.proc
// Older hosts have no authoritative task capability and their detached idle
// timer ignores background work. Keep their connection until explicit close.
if p != nil && p.link != nil && !p.link.tasksKnown {
s.mu.Unlock()
return
}
s.proc = nil
s.mu.Unlock()
if p != nil {
if p.link != nil {
p.detach()
} else {
p.close()
}
}
}
func (s *session) closeProcess() {
s.mu.Lock()
p := s.proc
+3
View File
@@ -141,6 +141,9 @@ func TestHostedDetachRetainsAnswerableInteractions(t *testing.T) {
}
wait(before)
before.Detach()
// Default lifetime is unbounded: keep the original upstream callback identity
// even across more than one host expiry tick while Pipelines is down.
time.Sleep(1200 * time.Millisecond)
after, err := Open(ctx)
if err != nil {
t.Fatal(err)
+4 -3
View File
@@ -55,8 +55,9 @@ type bridge struct {
detached time.Time
}
// Options describes one private, database-scoped bridge. MaxDetached bounds its
// lifetime without a Pipelines client, even with an unresolved approval.
// Options describes one private, database-scoped bridge. Positive timeout
// values opt into detached expiry; zero keeps the original upstream connection
// and its live sessions/callbacks until explicitly stopped.
type Options struct {
Dir, Upstream string
Idle, MaxDetached time.Duration
@@ -221,7 +222,7 @@ func (h *bridge) expire(idle, maxDetached time.Duration) {
return
case <-ticker.C:
h.mu.Lock()
expired := h.active == nil && (time.Since(h.detached) >= maxDetached || (len(h.pending) == 0 && time.Since(h.detached) >= idle))
expired := h.active == nil && ((maxDetached > 0 && time.Since(h.detached) >= maxDetached) || (idle > 0 && len(h.pending) == 0 && time.Since(h.detached) >= idle))
h.mu.Unlock()
if expired {
h.cancel()
+2 -3
View File
@@ -7,7 +7,6 @@ import (
"os"
"os/signal"
"syscall"
"time"
)
// Main is the detached bridge's command entry point.
@@ -15,8 +14,8 @@ func Main(args []string) int {
flags := flag.NewFlagSet("pipelines-codex-host", flag.ContinueOnError)
dir := flags.String("dir", "", "private per-server host directory")
upstream := flags.String("upstream", "", "existing Codex App Server Unix socket")
idle := flags.Duration("idle", 20*time.Minute, "exit after this long detached without pending interactions")
maxDetached := flags.Duration("max-detached", 24*time.Hour, "maximum detached lifetime, including pending interactions")
idle := flags.Duration("idle", 0, "optional detached idle expiry (0 keeps the upstream session connection)")
maxDetached := flags.Duration("max-detached", 0, "optional detached lifetime limit (0 keeps live sessions)")
if err := flags.Parse(args); err != nil {
return 2
}
@@ -0,0 +1,43 @@
package opencode
import (
"context"
"net/http"
"net/http/httptest"
"testing"
)
func TestOwnedIdleRequiresAuthoritativeBackgroundCompletion(t *testing.T) {
for _, tc := range []struct {
name, body string
status int
idle bool
}{
{"empty envelope", `{"data":[]}`, 200, true},
{"empty list", `[]`, 200, true},
{"exited", `{"data":[{"status":"exited"}]}`, 200, true},
{"running", `{"data":[{"status":"running"}]}`, 200, false},
{"missing status", `{"data":[{}]}`, 200, false},
{"missing membership", `{}`, 200, false},
{"null membership", `{"data":null}`, 200, false},
{"invalid response", `bad`, 200, false},
{"readback unavailable", ``, 503, false},
} {
t.Run(tc.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path == "/api/session/active" {
_, _ = w.Write([]byte(`{"data":{}}`))
return
}
w.WriteHeader(tc.status)
_, _ = w.Write([]byte(tc.body))
}))
defer server.Close()
c := &Connection{Backend: "v2", URL: server.URL, password: "test"}
if got := c.ownedIdle(context.Background()); got != tc.idle {
t.Fatalf("idle=%v want%v", got, tc.idle)
}
})
}
}
+30 -4
View File
@@ -426,8 +426,10 @@ func (c *Connection) ownedInfo(ctx context.Context) (int, bool) {
return info.PID, true
}
// ownedIdle reports whether no session is active on this process; any error
// counts as busy so active work is never stopped on a guess.
// ownedIdle requires both the foreground drain and background shells to be
// empty/finished. A completed turn can leave real shells running on this
// service; stopping it would kill those tasks across a server restart.
// Any missing, malformed or unavailable readback counts as busy.
func (c *Connection) ownedIdle(ctx context.Context) bool {
probe, cancel := context.WithTimeout(ctx, ownedStageProbeTimeout)
defer cancel()
@@ -438,11 +440,35 @@ func (c *Connection) ownedIdle(ctx context.Context) bool {
var active struct {
Data map[string]json.RawMessage `json:"data"`
}
return json.Unmarshal(data, &active) == nil && active.Data != nil && len(active.Data) == 0
if json.Unmarshal(data, &active) != nil || active.Data == nil || len(active.Data) != 0 {
return false
}
data, err = c.nativeRequest(probe, "get", "/api/shell", nil, "")
if err != nil {
return false
}
var shells []struct {
Status string `json:"status"`
}
var response struct {
Data json.RawMessage `json:"data"`
}
if json.Unmarshal(data, &response) == nil && response.Data != nil {
data = response.Data
}
if json.Unmarshal(data, &shells) != nil || shells == nil {
return false
}
for _, shell := range shells {
if shell.Status != "exited" {
return false
}
}
return true
}
// StopIdleStageService stops the owned stage service on server shutdown when no
// session is active on it. The registration (URL and password) is kept, so a
// foreground turn or background shell is active on it. The registration (URL and password) is kept, so a
// later start reuses the same endpoint and retained references stay valid. A
// busy service keeps running so restart recovery can interrupt it properly.
func StopIdleStageService() {
+128
View File
@@ -3,9 +3,12 @@ package opencode
import (
"context"
"encoding/json"
"fmt"
"git.a-wan.net/alan/pipelines/internal/protocol"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
@@ -63,6 +66,57 @@ func runFakeOpenCodeServe() {
}
_, _ = w.Write([]byte(`{"data":{}}`))
}))
// A real task in this independent service keeps producing output while no
// Pipelines adapter is connected. The marker is its explicit completion gate.
outputPath := os.Getenv("FAKE_SHELL_OUTPUT")
if outputPath != "" {
go func() {
for i := 0; ; i++ {
if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err != nil {
return
}
f, err := os.OpenFile(outputPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return
}
_, _ = fmt.Fprintln(f, i)
_ = f.Close()
time.Sleep(20 * time.Millisecond)
}
}()
}
shellInfo := func() map[string]any {
status := "exited"
if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err == nil {
status = "running"
}
return map[string]any{"id": "sh_background", "status": status, "metadata": map[string]string{"sessionID": "ses_background"}, "command": "background heartbeat"}
}
mux.HandleFunc("/api/shell", auth(func(w http.ResponseWriter) {
_, running := os.Stat(os.Getenv("FAKE_SHELL_FILE"))
_, ran := os.Stat(outputPath)
if running == nil || (outputPath != "" && ran == nil) {
_ = json.NewEncoder(w).Encode(map[string]any{"data": []any{shellInfo()}})
return
}
_, _ = w.Write([]byte(`{"data":[]}`))
}))
mux.HandleFunc("/api/shell/sh_background", auth(func(w http.ResponseWriter) { _ = json.NewEncoder(w).Encode(map[string]any{"data": shellInfo()}) }))
mux.HandleFunc("/api/shell/sh_background/output", func(w http.ResponseWriter, r *http.Request) {
auth(func(w http.ResponseWriter) {
data, _ := os.ReadFile(outputPath)
cursor, _ := strconv.Atoi(r.URL.Query().Get("cursor"))
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
if cursor > len(data) {
cursor = len(data)
}
end := cursor + limit
if end > len(data) {
end = len(data)
}
_ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"output": string(data[cursor:end]), "cursor": end, "size": len(data), "truncated": false}})
})(w, r)
})
mux.HandleFunc("/api/env", auth(func(w http.ResponseWriter) {
values := map[string]*string{}
for _, key := range strings.Split(os.Getenv("FAKE_ENV_KEYS"), ",") {
@@ -87,6 +141,7 @@ func fakeOwnedStageEnv(t *testing.T) string {
t.Setenv(OpenCodeServiceStateEnvKey, t.TempDir())
t.Setenv(fakeOpenCodeEnv, "1")
t.Setenv("FAKE_BUSY_FILE", filepath.Join(t.TempDir(), "busy"))
t.Setenv("FAKE_SHELL_FILE", filepath.Join(t.TempDir(), "background"))
for _, key := range []string{StageServiceEnvKey, StageToolOutputMaxBytesEnvKey, StageToolOutputMaxLinesEnvKey, StageCompactionEnvKey, StageConfigEnvKey, "OPENCODE_CONFIG_CONTENT", "FAKE_FAIL"} {
t.Setenv(key, "")
}
@@ -287,3 +342,76 @@ func waitGone(t *testing.T, pid int) {
}
t.Fatalf("process %d still running", pid)
}
// Foreground idle does not imply service idle. This is a process-level restart:
// the adapter closes, shutdown cleanup runs, the task keeps writing while the
// server is away, and a new adapter reattaches the same native shell/output.
func TestBackgroundOnlyStageSurvivesOwnerRestart(t *testing.T) {
executable := fakeOwnedStageEnv(t)
marker := os.Getenv("FAKE_SHELL_FILE")
if err := os.WriteFile(marker, nil, 0600); err != nil {
t.Fatal(err)
}
output := filepath.Join(t.TempDir(), "output")
t.Setenv("FAKE_SHELL_OUTPUT", output)
first, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
if err != nil {
t.Fatal(err)
}
pid := ownedPID(t, first)
ref, err := first.Reference()
if err != nil {
t.Fatal(err)
}
before := NewChat(first)
shells, err := before.Shells(context.Background(), "ses_background")
if err != nil || len(shells) != 1 {
t.Fatalf("native task: %+v %v", shells, err)
}
query := protocol.ChatShellOutputQuery{SessionID: "ses_background", ShellID: shells[0].ID, Limit: 4096}
page, err := before.ShellOutput(context.Background(), query)
if err != nil {
t.Fatal(err)
}
first.Close()
StopIdleStageService()
time.Sleep(100 * time.Millisecond)
second, err := FromReference(ref)
if err != nil {
t.Fatal(err)
}
defer second.Close()
if ownedPID(t, second) != pid {
t.Fatal("restart replaced the live background service")
}
after := NewChat(second)
got, err := after.Shells(context.Background(), "ses_background")
if err != nil || len(got) != 1 || got[0].ID != shells[0].ID || got[0].Status != protocol.ChatShellRunning {
t.Fatalf("reattached task %+v %v", got, err)
}
continued, err := after.ShellOutput(context.Background(), query)
if err != nil || continued.Size <= page.Size {
t.Fatalf("task did not progress during restart: %+v -> %+v %v", page, continued, err)
}
t.Logf("backend PID=%d, native shell=%s, output bytes before=%d after=%d", pid, got[0].ID, page.Size, continued.Size)
// Configuration replacement and idle timeout must protect the same task too.
t.Setenv(StageToolOutputMaxBytesEnvKey, "1234")
changed, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
if err != nil || ownedPID(t, changed) != pid {
t.Fatalf("background service replaced on config update: %v", err)
}
path, _ := StageServiceRegistrationPath()
ownedStageLocksMu.Lock()
ownedStageUsed[path] = time.Now().Add(-time.Hour)
ownedStageLocksMu.Unlock()
if stopIdleOwnedStage(path, time.Minute) {
t.Fatal("idle reaper stopped background task")
}
_ = os.Remove(marker)
terminal, err := after.ShellOutput(context.Background(), query)
if err != nil || terminal.Shell.ID != query.ShellID || terminal.Shell.Status != protocol.ChatShellExited {
t.Fatalf("completion not observable: %+v %v", terminal, err)
}
StopIdleStageService()
waitGone(t, pid)
}