diff --git a/docs/chat-backends.md b/docs/chat-backends.md index 8ea96ae2..fb1768d6 100644 --- a/docs/chat-backends.md +++ b/docs/chat-backends.md @@ -96,8 +96,8 @@ Ordinary notifications are not spooled; the existing thread readback repairs gap State is bounded to 128 callbacks / 8 MiB and 256 in-flight client calls, with a 16 MiB frame limit. Exceeding pending bounds closes the bridge without granting, -declining or otherwise auto-answering an interaction. A detached bridge expires -after 20 minutes with no pending interactions, or after 24 hours even if pending. +declining or otherwise auto-answering an interaction. A detached bridge keeps its original upstream connection indefinitely by default. +Only explicitly supplied positive `-idle`/`-max-detached` limits enable expiry. An upstream App Server restart, a bridge crash, or expiry loses its connection- bound callbacks: no disk JSON can recreate them on a new connection. The initial deployment cannot recover requests already owned by the old direct connection. @@ -161,16 +161,20 @@ The host stops the CLI (EOF, then SIGINT, then SIGKILL to its process group) and removes its files only when one of these happens: - The operator closes the chat (`chat_close`). -- The server's idle release or live-process cap fires. +- A genuinely idle unhosted CLI is released. For hosted sessions, server cache + eviction only detaches; the host owns authoritative background-task liveness. - The CLI exits; the server then gets its stderr. -- With no server attached, the CLI has been idle for the same 20 minutes, or - no server has attached for 24 hours. +- With no server attached, the CLI has been genuinely idle for 20 minutes, + with no foreground turn, pending interaction, or native background task. + There is no default hard detached lifetime limit. -If the host itself dies, the server first tries to reconnect once. If that fails, -it stops the orphaned CLI's process group (only when that pid's command line -still names the session) and removes the host files. A running turn ends with -the notice "the Claude Code host stopped unexpectedly; the next prompt resumes -this session with --resume". The next prompt then resumes the session. +If the control connection is lost, the server first tries to reconnect. A verified +live host is preserved when its socket is temporarily unavailable, and another +CLI is not started alongside it. Dead host recovery verifies process identity +before stopping an orphan. On attachment, the host's authoritative task snapshot +restores running task identities even after older spool records were trimmed. +Legacy hosts without that snapshot remain attached during cache eviction, so +their older detached idle timers cannot mistake background work for an idle CLI. Limits: diff --git a/internal/claude/background_tasks_test.go b/internal/claude/background_tasks_test.go index 31690a18..e620f760 100644 --- a/internal/claude/background_tasks_test.go +++ b/internal/claude/background_tasks_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "testing" + "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) @@ -150,3 +151,51 @@ func TestProcessExitEndsBackgroundTasks(t *testing.T) { t.Fatal("no shell exit was published") } } + +func TestObserverReleaseDoesNotForgetOrReapRunningNativeTasks(t *testing.T) { + chat, s := backgroundSession(t) + defer chat.Close() + s.handleLine(nil, toolUse(t, "toolu_bg", "Bash", map[string]any{"command": "quiet background work", "run_in_background": true})) + s.handleLine(nil, toolResult(t, "toolu_bg", "Command running in background with ID: native-bg")) + s.handleLine(nil, toolUse(t, "toolu_done", "Bash", map[string]any{"command": "completed foreground"})) + s.handleLine(nil, toolResult(t, "toolu_done", "done")) + proc := newProcess(s, "") + proc.markReady() + s.mu.Lock() + s.proc = proc + s.running = false + s.lastUsed = time.Now().Add(-time.Hour) + s.mu.Unlock() + chat.ReleaseShells(s.id) + if shells := running(t, chat); len(shells) != 1 || shells[0].ID != shellID("toolu_bg") { + t.Fatalf("observer expiry lost task identity: %+v", shells) + } + chat.reapIdleAt(time.Now()) + s.mu.Lock() + same := s.proc == proc + s.proc = nil + s.mu.Unlock() + if !same { + t.Fatal("idle cache eviction stopped active background work") + } +} + +func TestLegacyHostRemainsAttachedDuringIdleCacheEviction(t *testing.T) { + chat, s := backgroundSession(t) + defer chat.Close() + proc := newProcess(s, "") + proc.markReady() + proc.link = &hostLink{} // No authoritative task snapshot capability. + s.mu.Lock() + s.proc = proc + s.lastUsed = time.Now().Add(-time.Hour) + s.mu.Unlock() + chat.reapIdleAt(time.Now()) + s.mu.Lock() + same := s.proc == proc + s.proc = nil + s.mu.Unlock() + if !same { + t.Fatal("legacy host was detached despite unknown background liveness") + } +} diff --git a/internal/claude/claude.go b/internal/claude/claude.go index 92c61047..2b2e4cc8 100644 --- a/internal/claude/claude.go +++ b/internal/claude/claude.go @@ -161,13 +161,27 @@ func (c *Chat) Close() { } } -// ReleaseShells drops a session's retained shell snapshot after the last -// observer leaves. A later attach re-reads the authoritative transcript. +// ReleaseShells releases completed display snapshots, never liveness for +// running tasks. A room/observer expiry is not permission to stop native work. func (c *Chat) ReleaseShells(id string) { if s := c.session(id); s != nil { s.mu.Lock() - s.shells = map[string]*shellSnapshot{} - s.shellOrder = nil + kept := make(map[string]*shellSnapshot) + ids := map[string]bool{} + for key, task := range s.shells { + if task.shell.Status == protocol.ChatShellRunning { + kept[key] = task + ids[task.shell.ID] = true + } + } + var order []string + for _, id := range s.shellOrder { + if ids[id] { + order = append(order, id) + } + } + + s.shells, s.shellOrder = kept, order s.mu.Unlock() } } @@ -250,37 +264,46 @@ func (c *Chat) reapIdle() { return case <-ticker.C: } - now := time.Now() - type candidate struct { - s *session - used time.Time - alive bool + c.reapIdleAt(time.Now()) + } +} + +func (c *Chat) reapIdleAt(now time.Time) { + type candidate struct { + s *session + used time.Time + } + var live []candidate + c.mu.Lock() + for _, s := range c.sessions { + s.mu.Lock() + alive := s.proc != nil + used := s.lastUsed + s.mu.Unlock() + if alive { + live = append(live, candidate{s: s, used: used}) } - var live []candidate - c.mu.Lock() - for _, s := range c.sessions { - s.mu.Lock() - alive := s.proc != nil - used := s.lastUsed - s.mu.Unlock() - if alive { - live = append(live, candidate{s: s, used: used}) - } - } - c.mu.Unlock() - sort.Slice(live, func(i, j int) bool { return live[i].used.Before(live[j].used) }) - for _, entry := range live { - entry.s.mu.Lock() - excess := len(live) > maxLiveProcesses && !entry.s.running - idle := !entry.s.running && len(entry.s.subs) == 0 && now.Sub(entry.used) > idleProcessExpiry - entry.s.mu.Unlock() - if excess || idle { - entry.s.closeProcess() - // Recompute the live count on the next tick after a close. - live = live[1:] + } + c.mu.Unlock() + sort.Slice(live, func(i, j int) bool { return live[i].used.Before(live[j].used) }) + for _, entry := range live { + entry.s.mu.Lock() + busy := entry.s.running + for _, task := range entry.s.shells { + if task.shell.Status == protocol.ChatShellRunning { + busy = true break } } + excess := len(live) > maxLiveProcesses && !busy + idle := !busy && len(entry.s.subs) == 0 && now.Sub(entry.used) > idleProcessExpiry + entry.s.mu.Unlock() + if excess || idle { + entry.s.releaseIdleProcess() + // Recompute the live count on the next tick after a close. + live = live[1:] + break + } } } @@ -443,6 +466,39 @@ func (s *session) unsubscribe(ch chan protocol.ChatEventFrame) { s.mu.Unlock() } +// The cache reaper may relinquish an idle control connection, not stop a +// hosted session. Only the host knows authoritative tasks beyond spool replay; +// an explicit operator close remains the hard-close path below. +func (s *session) releaseIdleProcess() { + s.mu.Lock() + if s.running || len(s.subs) > 0 { + s.mu.Unlock() + return + } + for _, task := range s.shells { + if task.shell.Status == protocol.ChatShellRunning { + s.mu.Unlock() + return + } + } + p := s.proc + // Older hosts have no authoritative task capability and their detached idle + // timer ignores background work. Keep their connection until explicit close. + if p != nil && p.link != nil && !p.link.tasksKnown { + s.mu.Unlock() + return + } + s.proc = nil + s.mu.Unlock() + if p != nil { + if p.link != nil { + p.detach() + } else { + p.close() + } + } +} + func (s *session) closeProcess() { s.mu.Lock() p := s.proc diff --git a/internal/codex/hosted_test.go b/internal/codex/hosted_test.go index 05e4388d..74308a4f 100644 --- a/internal/codex/hosted_test.go +++ b/internal/codex/hosted_test.go @@ -141,6 +141,9 @@ func TestHostedDetachRetainsAnswerableInteractions(t *testing.T) { } wait(before) before.Detach() + // Default lifetime is unbounded: keep the original upstream callback identity + // even across more than one host expiry tick while Pipelines is down. + time.Sleep(1200 * time.Millisecond) after, err := Open(ctx) if err != nil { t.Fatal(err) diff --git a/internal/codexhost/host.go b/internal/codexhost/host.go index f2b031b2..040c6d62 100644 --- a/internal/codexhost/host.go +++ b/internal/codexhost/host.go @@ -55,8 +55,9 @@ type bridge struct { detached time.Time } -// Options describes one private, database-scoped bridge. MaxDetached bounds its -// lifetime without a Pipelines client, even with an unresolved approval. +// Options describes one private, database-scoped bridge. Positive timeout +// values opt into detached expiry; zero keeps the original upstream connection +// and its live sessions/callbacks until explicitly stopped. type Options struct { Dir, Upstream string Idle, MaxDetached time.Duration @@ -221,7 +222,7 @@ func (h *bridge) expire(idle, maxDetached time.Duration) { return case <-ticker.C: h.mu.Lock() - expired := h.active == nil && (time.Since(h.detached) >= maxDetached || (len(h.pending) == 0 && time.Since(h.detached) >= idle)) + expired := h.active == nil && ((maxDetached > 0 && time.Since(h.detached) >= maxDetached) || (idle > 0 && len(h.pending) == 0 && time.Since(h.detached) >= idle)) h.mu.Unlock() if expired { h.cancel() diff --git a/internal/codexhost/main.go b/internal/codexhost/main.go index 8b8e8755..dd3d58b6 100644 --- a/internal/codexhost/main.go +++ b/internal/codexhost/main.go @@ -7,7 +7,6 @@ import ( "os" "os/signal" "syscall" - "time" ) // Main is the detached bridge's command entry point. @@ -15,8 +14,8 @@ func Main(args []string) int { flags := flag.NewFlagSet("pipelines-codex-host", flag.ContinueOnError) dir := flags.String("dir", "", "private per-server host directory") upstream := flags.String("upstream", "", "existing Codex App Server Unix socket") - idle := flags.Duration("idle", 20*time.Minute, "exit after this long detached without pending interactions") - maxDetached := flags.Duration("max-detached", 24*time.Hour, "maximum detached lifetime, including pending interactions") + idle := flags.Duration("idle", 0, "optional detached idle expiry (0 keeps the upstream session connection)") + maxDetached := flags.Duration("max-detached", 0, "optional detached lifetime limit (0 keeps live sessions)") if err := flags.Parse(args); err != nil { return 2 } diff --git a/internal/opencode/service_idle_tasks_test.go b/internal/opencode/service_idle_tasks_test.go new file mode 100644 index 00000000..e0249b18 --- /dev/null +++ b/internal/opencode/service_idle_tasks_test.go @@ -0,0 +1,43 @@ +package opencode + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestOwnedIdleRequiresAuthoritativeBackgroundCompletion(t *testing.T) { + for _, tc := range []struct { + name, body string + status int + idle bool + }{ + {"empty envelope", `{"data":[]}`, 200, true}, + {"empty list", `[]`, 200, true}, + {"exited", `{"data":[{"status":"exited"}]}`, 200, true}, + {"running", `{"data":[{"status":"running"}]}`, 200, false}, + {"missing status", `{"data":[{}]}`, 200, false}, + {"missing membership", `{}`, 200, false}, + {"null membership", `{"data":null}`, 200, false}, + {"invalid response", `bad`, 200, false}, + {"readback unavailable", ``, 503, false}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/api/session/active" { + _, _ = w.Write([]byte(`{"data":{}}`)) + return + } + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + defer server.Close() + c := &Connection{Backend: "v2", URL: server.URL, password: "test"} + if got := c.ownedIdle(context.Background()); got != tc.idle { + t.Fatalf("idle=%v want%v", got, tc.idle) + } + }) + } +} diff --git a/internal/opencode/stage_owned.go b/internal/opencode/stage_owned.go index ac0aa4a4..f325f694 100644 --- a/internal/opencode/stage_owned.go +++ b/internal/opencode/stage_owned.go @@ -426,8 +426,10 @@ func (c *Connection) ownedInfo(ctx context.Context) (int, bool) { return info.PID, true } -// ownedIdle reports whether no session is active on this process; any error -// counts as busy so active work is never stopped on a guess. +// ownedIdle requires both the foreground drain and background shells to be +// empty/finished. A completed turn can leave real shells running on this +// service; stopping it would kill those tasks across a server restart. +// Any missing, malformed or unavailable readback counts as busy. func (c *Connection) ownedIdle(ctx context.Context) bool { probe, cancel := context.WithTimeout(ctx, ownedStageProbeTimeout) defer cancel() @@ -438,11 +440,35 @@ func (c *Connection) ownedIdle(ctx context.Context) bool { var active struct { Data map[string]json.RawMessage `json:"data"` } - return json.Unmarshal(data, &active) == nil && active.Data != nil && len(active.Data) == 0 + if json.Unmarshal(data, &active) != nil || active.Data == nil || len(active.Data) != 0 { + return false + } + data, err = c.nativeRequest(probe, "get", "/api/shell", nil, "") + if err != nil { + return false + } + var shells []struct { + Status string `json:"status"` + } + var response struct { + Data json.RawMessage `json:"data"` + } + if json.Unmarshal(data, &response) == nil && response.Data != nil { + data = response.Data + } + if json.Unmarshal(data, &shells) != nil || shells == nil { + return false + } + for _, shell := range shells { + if shell.Status != "exited" { + return false + } + } + return true } // StopIdleStageService stops the owned stage service on server shutdown when no -// session is active on it. The registration (URL and password) is kept, so a +// foreground turn or background shell is active on it. The registration (URL and password) is kept, so a // later start reuses the same endpoint and retained references stay valid. A // busy service keeps running so restart recovery can interrupt it properly. func StopIdleStageService() { diff --git a/internal/opencode/stage_owned_test.go b/internal/opencode/stage_owned_test.go index e3e368ce..5034fefa 100644 --- a/internal/opencode/stage_owned_test.go +++ b/internal/opencode/stage_owned_test.go @@ -3,9 +3,12 @@ package opencode import ( "context" "encoding/json" + "fmt" + "git.a-wan.net/alan/pipelines/internal/protocol" "net/http" "os" "path/filepath" + "strconv" "strings" "syscall" "testing" @@ -63,6 +66,57 @@ func runFakeOpenCodeServe() { } _, _ = w.Write([]byte(`{"data":{}}`)) })) + // A real task in this independent service keeps producing output while no + // Pipelines adapter is connected. The marker is its explicit completion gate. + outputPath := os.Getenv("FAKE_SHELL_OUTPUT") + if outputPath != "" { + go func() { + for i := 0; ; i++ { + if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err != nil { + return + } + f, err := os.OpenFile(outputPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return + } + _, _ = fmt.Fprintln(f, i) + _ = f.Close() + time.Sleep(20 * time.Millisecond) + } + }() + } + shellInfo := func() map[string]any { + status := "exited" + if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err == nil { + status = "running" + } + return map[string]any{"id": "sh_background", "status": status, "metadata": map[string]string{"sessionID": "ses_background"}, "command": "background heartbeat"} + } + mux.HandleFunc("/api/shell", auth(func(w http.ResponseWriter) { + _, running := os.Stat(os.Getenv("FAKE_SHELL_FILE")) + _, ran := os.Stat(outputPath) + if running == nil || (outputPath != "" && ran == nil) { + _ = json.NewEncoder(w).Encode(map[string]any{"data": []any{shellInfo()}}) + return + } + _, _ = w.Write([]byte(`{"data":[]}`)) + })) + mux.HandleFunc("/api/shell/sh_background", auth(func(w http.ResponseWriter) { _ = json.NewEncoder(w).Encode(map[string]any{"data": shellInfo()}) })) + mux.HandleFunc("/api/shell/sh_background/output", func(w http.ResponseWriter, r *http.Request) { + auth(func(w http.ResponseWriter) { + data, _ := os.ReadFile(outputPath) + cursor, _ := strconv.Atoi(r.URL.Query().Get("cursor")) + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + if cursor > len(data) { + cursor = len(data) + } + end := cursor + limit + if end > len(data) { + end = len(data) + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"output": string(data[cursor:end]), "cursor": end, "size": len(data), "truncated": false}}) + })(w, r) + }) mux.HandleFunc("/api/env", auth(func(w http.ResponseWriter) { values := map[string]*string{} for _, key := range strings.Split(os.Getenv("FAKE_ENV_KEYS"), ",") { @@ -87,6 +141,7 @@ func fakeOwnedStageEnv(t *testing.T) string { t.Setenv(OpenCodeServiceStateEnvKey, t.TempDir()) t.Setenv(fakeOpenCodeEnv, "1") t.Setenv("FAKE_BUSY_FILE", filepath.Join(t.TempDir(), "busy")) + t.Setenv("FAKE_SHELL_FILE", filepath.Join(t.TempDir(), "background")) for _, key := range []string{StageServiceEnvKey, StageToolOutputMaxBytesEnvKey, StageToolOutputMaxLinesEnvKey, StageCompactionEnvKey, StageConfigEnvKey, "OPENCODE_CONFIG_CONTENT", "FAKE_FAIL"} { t.Setenv(key, "") } @@ -287,3 +342,76 @@ func waitGone(t *testing.T, pid int) { } t.Fatalf("process %d still running", pid) } + +// Foreground idle does not imply service idle. This is a process-level restart: +// the adapter closes, shutdown cleanup runs, the task keeps writing while the +// server is away, and a new adapter reattaches the same native shell/output. +func TestBackgroundOnlyStageSurvivesOwnerRestart(t *testing.T) { + executable := fakeOwnedStageEnv(t) + marker := os.Getenv("FAKE_SHELL_FILE") + if err := os.WriteFile(marker, nil, 0600); err != nil { + t.Fatal(err) + } + output := filepath.Join(t.TempDir(), "output") + t.Setenv("FAKE_SHELL_OUTPUT", output) + first, err := OpenStage(context.Background(), executable, t.TempDir(), "v2") + if err != nil { + t.Fatal(err) + } + pid := ownedPID(t, first) + ref, err := first.Reference() + if err != nil { + t.Fatal(err) + } + before := NewChat(first) + shells, err := before.Shells(context.Background(), "ses_background") + if err != nil || len(shells) != 1 { + t.Fatalf("native task: %+v %v", shells, err) + } + query := protocol.ChatShellOutputQuery{SessionID: "ses_background", ShellID: shells[0].ID, Limit: 4096} + page, err := before.ShellOutput(context.Background(), query) + if err != nil { + t.Fatal(err) + } + first.Close() + StopIdleStageService() + time.Sleep(100 * time.Millisecond) + second, err := FromReference(ref) + if err != nil { + t.Fatal(err) + } + defer second.Close() + if ownedPID(t, second) != pid { + t.Fatal("restart replaced the live background service") + } + after := NewChat(second) + got, err := after.Shells(context.Background(), "ses_background") + if err != nil || len(got) != 1 || got[0].ID != shells[0].ID || got[0].Status != protocol.ChatShellRunning { + t.Fatalf("reattached task %+v %v", got, err) + } + continued, err := after.ShellOutput(context.Background(), query) + if err != nil || continued.Size <= page.Size { + t.Fatalf("task did not progress during restart: %+v -> %+v %v", page, continued, err) + } + t.Logf("backend PID=%d, native shell=%s, output bytes before=%d after=%d", pid, got[0].ID, page.Size, continued.Size) + // Configuration replacement and idle timeout must protect the same task too. + t.Setenv(StageToolOutputMaxBytesEnvKey, "1234") + changed, err := OpenStage(context.Background(), executable, t.TempDir(), "v2") + if err != nil || ownedPID(t, changed) != pid { + t.Fatalf("background service replaced on config update: %v", err) + } + path, _ := StageServiceRegistrationPath() + ownedStageLocksMu.Lock() + ownedStageUsed[path] = time.Now().Add(-time.Hour) + ownedStageLocksMu.Unlock() + if stopIdleOwnedStage(path, time.Minute) { + t.Fatal("idle reaper stopped background task") + } + _ = os.Remove(marker) + terminal, err := after.ShellOutput(context.Background(), query) + if err != nil || terminal.Shell.ID != query.ShellID || terminal.Shell.Status != protocol.ChatShellExited { + t.Fatalf("completion not observable: %+v %v", terminal, err) + } + StopIdleStageService() + waitGone(t, pid) +}