Preserve independent background sessions through owner shutdown and idle cleanup
This commit is contained in:
+14
-10
@@ -96,8 +96,8 @@ Ordinary notifications are not spooled; the existing thread readback repairs gap
|
||||
|
||||
State is bounded to 128 callbacks / 8 MiB and 256 in-flight client calls, with a
|
||||
16 MiB frame limit. Exceeding pending bounds closes the bridge without granting,
|
||||
declining or otherwise auto-answering an interaction. A detached bridge expires
|
||||
after 20 minutes with no pending interactions, or after 24 hours even if pending.
|
||||
declining or otherwise auto-answering an interaction. A detached bridge keeps its original upstream connection indefinitely by default.
|
||||
Only explicitly supplied positive `-idle`/`-max-detached` limits enable expiry.
|
||||
An upstream App Server restart, a bridge crash, or expiry loses its connection-
|
||||
bound callbacks: no disk JSON can recreate them on a new connection. The initial
|
||||
deployment cannot recover requests already owned by the old direct connection.
|
||||
@@ -161,16 +161,20 @@ The host stops the CLI (EOF, then SIGINT, then SIGKILL to its process group) and
|
||||
removes its files only when one of these happens:
|
||||
|
||||
- The operator closes the chat (`chat_close`).
|
||||
- The server's idle release or live-process cap fires.
|
||||
- A genuinely idle unhosted CLI is released. For hosted sessions, server cache
|
||||
eviction only detaches; the host owns authoritative background-task liveness.
|
||||
- The CLI exits; the server then gets its stderr.
|
||||
- With no server attached, the CLI has been idle for the same 20 minutes, or
|
||||
no server has attached for 24 hours.
|
||||
- With no server attached, the CLI has been genuinely idle for 20 minutes,
|
||||
with no foreground turn, pending interaction, or native background task.
|
||||
There is no default hard detached lifetime limit.
|
||||
|
||||
If the host itself dies, the server first tries to reconnect once. If that fails,
|
||||
it stops the orphaned CLI's process group (only when that pid's command line
|
||||
still names the session) and removes the host files. A running turn ends with
|
||||
the notice "the Claude Code host stopped unexpectedly; the next prompt resumes
|
||||
this session with --resume". The next prompt then resumes the session.
|
||||
If the control connection is lost, the server first tries to reconnect. A verified
|
||||
live host is preserved when its socket is temporarily unavailable, and another
|
||||
CLI is not started alongside it. Dead host recovery verifies process identity
|
||||
before stopping an orphan. On attachment, the host's authoritative task snapshot
|
||||
restores running task identities even after older spool records were trimmed.
|
||||
Legacy hosts without that snapshot remain attached during cache eviction, so
|
||||
their older detached idle timers cannot mistake background work for an idle CLI.
|
||||
|
||||
Limits:
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
@@ -150,3 +151,51 @@ func TestProcessExitEndsBackgroundTasks(t *testing.T) {
|
||||
t.Fatal("no shell exit was published")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverReleaseDoesNotForgetOrReapRunningNativeTasks(t *testing.T) {
|
||||
chat, s := backgroundSession(t)
|
||||
defer chat.Close()
|
||||
s.handleLine(nil, toolUse(t, "toolu_bg", "Bash", map[string]any{"command": "quiet background work", "run_in_background": true}))
|
||||
s.handleLine(nil, toolResult(t, "toolu_bg", "Command running in background with ID: native-bg"))
|
||||
s.handleLine(nil, toolUse(t, "toolu_done", "Bash", map[string]any{"command": "completed foreground"}))
|
||||
s.handleLine(nil, toolResult(t, "toolu_done", "done"))
|
||||
proc := newProcess(s, "")
|
||||
proc.markReady()
|
||||
s.mu.Lock()
|
||||
s.proc = proc
|
||||
s.running = false
|
||||
s.lastUsed = time.Now().Add(-time.Hour)
|
||||
s.mu.Unlock()
|
||||
chat.ReleaseShells(s.id)
|
||||
if shells := running(t, chat); len(shells) != 1 || shells[0].ID != shellID("toolu_bg") {
|
||||
t.Fatalf("observer expiry lost task identity: %+v", shells)
|
||||
}
|
||||
chat.reapIdleAt(time.Now())
|
||||
s.mu.Lock()
|
||||
same := s.proc == proc
|
||||
s.proc = nil
|
||||
s.mu.Unlock()
|
||||
if !same {
|
||||
t.Fatal("idle cache eviction stopped active background work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyHostRemainsAttachedDuringIdleCacheEviction(t *testing.T) {
|
||||
chat, s := backgroundSession(t)
|
||||
defer chat.Close()
|
||||
proc := newProcess(s, "")
|
||||
proc.markReady()
|
||||
proc.link = &hostLink{} // No authoritative task snapshot capability.
|
||||
s.mu.Lock()
|
||||
s.proc = proc
|
||||
s.lastUsed = time.Now().Add(-time.Hour)
|
||||
s.mu.Unlock()
|
||||
chat.reapIdleAt(time.Now())
|
||||
s.mu.Lock()
|
||||
same := s.proc == proc
|
||||
s.proc = nil
|
||||
s.mu.Unlock()
|
||||
if !same {
|
||||
t.Fatal("legacy host was detached despite unknown background liveness")
|
||||
}
|
||||
}
|
||||
|
||||
+87
-31
@@ -161,13 +161,27 @@ func (c *Chat) Close() {
|
||||
}
|
||||
}
|
||||
|
||||
// ReleaseShells drops a session's retained shell snapshot after the last
|
||||
// observer leaves. A later attach re-reads the authoritative transcript.
|
||||
// ReleaseShells releases completed display snapshots, never liveness for
|
||||
// running tasks. A room/observer expiry is not permission to stop native work.
|
||||
func (c *Chat) ReleaseShells(id string) {
|
||||
if s := c.session(id); s != nil {
|
||||
s.mu.Lock()
|
||||
s.shells = map[string]*shellSnapshot{}
|
||||
s.shellOrder = nil
|
||||
kept := make(map[string]*shellSnapshot)
|
||||
ids := map[string]bool{}
|
||||
for key, task := range s.shells {
|
||||
if task.shell.Status == protocol.ChatShellRunning {
|
||||
kept[key] = task
|
||||
ids[task.shell.ID] = true
|
||||
}
|
||||
}
|
||||
var order []string
|
||||
for _, id := range s.shellOrder {
|
||||
if ids[id] {
|
||||
order = append(order, id)
|
||||
}
|
||||
}
|
||||
|
||||
s.shells, s.shellOrder = kept, order
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
@@ -250,37 +264,46 @@ func (c *Chat) reapIdle() {
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
now := time.Now()
|
||||
type candidate struct {
|
||||
s *session
|
||||
used time.Time
|
||||
alive bool
|
||||
c.reapIdleAt(time.Now())
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Chat) reapIdleAt(now time.Time) {
|
||||
type candidate struct {
|
||||
s *session
|
||||
used time.Time
|
||||
}
|
||||
var live []candidate
|
||||
c.mu.Lock()
|
||||
for _, s := range c.sessions {
|
||||
s.mu.Lock()
|
||||
alive := s.proc != nil
|
||||
used := s.lastUsed
|
||||
s.mu.Unlock()
|
||||
if alive {
|
||||
live = append(live, candidate{s: s, used: used})
|
||||
}
|
||||
var live []candidate
|
||||
c.mu.Lock()
|
||||
for _, s := range c.sessions {
|
||||
s.mu.Lock()
|
||||
alive := s.proc != nil
|
||||
used := s.lastUsed
|
||||
s.mu.Unlock()
|
||||
if alive {
|
||||
live = append(live, candidate{s: s, used: used})
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
sort.Slice(live, func(i, j int) bool { return live[i].used.Before(live[j].used) })
|
||||
for _, entry := range live {
|
||||
entry.s.mu.Lock()
|
||||
excess := len(live) > maxLiveProcesses && !entry.s.running
|
||||
idle := !entry.s.running && len(entry.s.subs) == 0 && now.Sub(entry.used) > idleProcessExpiry
|
||||
entry.s.mu.Unlock()
|
||||
if excess || idle {
|
||||
entry.s.closeProcess()
|
||||
// Recompute the live count on the next tick after a close.
|
||||
live = live[1:]
|
||||
}
|
||||
c.mu.Unlock()
|
||||
sort.Slice(live, func(i, j int) bool { return live[i].used.Before(live[j].used) })
|
||||
for _, entry := range live {
|
||||
entry.s.mu.Lock()
|
||||
busy := entry.s.running
|
||||
for _, task := range entry.s.shells {
|
||||
if task.shell.Status == protocol.ChatShellRunning {
|
||||
busy = true
|
||||
break
|
||||
}
|
||||
}
|
||||
excess := len(live) > maxLiveProcesses && !busy
|
||||
idle := !busy && len(entry.s.subs) == 0 && now.Sub(entry.used) > idleProcessExpiry
|
||||
entry.s.mu.Unlock()
|
||||
if excess || idle {
|
||||
entry.s.releaseIdleProcess()
|
||||
// Recompute the live count on the next tick after a close.
|
||||
live = live[1:]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,6 +466,39 @@ func (s *session) unsubscribe(ch chan protocol.ChatEventFrame) {
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// The cache reaper may relinquish an idle control connection, not stop a
|
||||
// hosted session. Only the host knows authoritative tasks beyond spool replay;
|
||||
// an explicit operator close remains the hard-close path below.
|
||||
func (s *session) releaseIdleProcess() {
|
||||
s.mu.Lock()
|
||||
if s.running || len(s.subs) > 0 {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
for _, task := range s.shells {
|
||||
if task.shell.Status == protocol.ChatShellRunning {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
p := s.proc
|
||||
// Older hosts have no authoritative task capability and their detached idle
|
||||
// timer ignores background work. Keep their connection until explicit close.
|
||||
if p != nil && p.link != nil && !p.link.tasksKnown {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.proc = nil
|
||||
s.mu.Unlock()
|
||||
if p != nil {
|
||||
if p.link != nil {
|
||||
p.detach()
|
||||
} else {
|
||||
p.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *session) closeProcess() {
|
||||
s.mu.Lock()
|
||||
p := s.proc
|
||||
|
||||
@@ -141,6 +141,9 @@ func TestHostedDetachRetainsAnswerableInteractions(t *testing.T) {
|
||||
}
|
||||
wait(before)
|
||||
before.Detach()
|
||||
// Default lifetime is unbounded: keep the original upstream callback identity
|
||||
// even across more than one host expiry tick while Pipelines is down.
|
||||
time.Sleep(1200 * time.Millisecond)
|
||||
after, err := Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -55,8 +55,9 @@ type bridge struct {
|
||||
detached time.Time
|
||||
}
|
||||
|
||||
// Options describes one private, database-scoped bridge. MaxDetached bounds its
|
||||
// lifetime without a Pipelines client, even with an unresolved approval.
|
||||
// Options describes one private, database-scoped bridge. Positive timeout
|
||||
// values opt into detached expiry; zero keeps the original upstream connection
|
||||
// and its live sessions/callbacks until explicitly stopped.
|
||||
type Options struct {
|
||||
Dir, Upstream string
|
||||
Idle, MaxDetached time.Duration
|
||||
@@ -221,7 +222,7 @@ func (h *bridge) expire(idle, maxDetached time.Duration) {
|
||||
return
|
||||
case <-ticker.C:
|
||||
h.mu.Lock()
|
||||
expired := h.active == nil && (time.Since(h.detached) >= maxDetached || (len(h.pending) == 0 && time.Since(h.detached) >= idle))
|
||||
expired := h.active == nil && ((maxDetached > 0 && time.Since(h.detached) >= maxDetached) || (idle > 0 && len(h.pending) == 0 && time.Since(h.detached) >= idle))
|
||||
h.mu.Unlock()
|
||||
if expired {
|
||||
h.cancel()
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Main is the detached bridge's command entry point.
|
||||
@@ -15,8 +14,8 @@ func Main(args []string) int {
|
||||
flags := flag.NewFlagSet("pipelines-codex-host", flag.ContinueOnError)
|
||||
dir := flags.String("dir", "", "private per-server host directory")
|
||||
upstream := flags.String("upstream", "", "existing Codex App Server Unix socket")
|
||||
idle := flags.Duration("idle", 20*time.Minute, "exit after this long detached without pending interactions")
|
||||
maxDetached := flags.Duration("max-detached", 24*time.Hour, "maximum detached lifetime, including pending interactions")
|
||||
idle := flags.Duration("idle", 0, "optional detached idle expiry (0 keeps the upstream session connection)")
|
||||
maxDetached := flags.Duration("max-detached", 0, "optional detached lifetime limit (0 keeps live sessions)")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOwnedIdleRequiresAuthoritativeBackgroundCompletion(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, body string
|
||||
status int
|
||||
idle bool
|
||||
}{
|
||||
{"empty envelope", `{"data":[]}`, 200, true},
|
||||
{"empty list", `[]`, 200, true},
|
||||
{"exited", `{"data":[{"status":"exited"}]}`, 200, true},
|
||||
{"running", `{"data":[{"status":"running"}]}`, 200, false},
|
||||
{"missing status", `{"data":[{}]}`, 200, false},
|
||||
{"missing membership", `{}`, 200, false},
|
||||
{"null membership", `{"data":null}`, 200, false},
|
||||
{"invalid response", `bad`, 200, false},
|
||||
{"readback unavailable", ``, 503, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Path == "/api/session/active" {
|
||||
_, _ = w.Write([]byte(`{"data":{}}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(tc.status)
|
||||
_, _ = w.Write([]byte(tc.body))
|
||||
}))
|
||||
defer server.Close()
|
||||
c := &Connection{Backend: "v2", URL: server.URL, password: "test"}
|
||||
if got := c.ownedIdle(context.Background()); got != tc.idle {
|
||||
t.Fatalf("idle=%v want%v", got, tc.idle)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -426,8 +426,10 @@ func (c *Connection) ownedInfo(ctx context.Context) (int, bool) {
|
||||
return info.PID, true
|
||||
}
|
||||
|
||||
// ownedIdle reports whether no session is active on this process; any error
|
||||
// counts as busy so active work is never stopped on a guess.
|
||||
// ownedIdle requires both the foreground drain and background shells to be
|
||||
// empty/finished. A completed turn can leave real shells running on this
|
||||
// service; stopping it would kill those tasks across a server restart.
|
||||
// Any missing, malformed or unavailable readback counts as busy.
|
||||
func (c *Connection) ownedIdle(ctx context.Context) bool {
|
||||
probe, cancel := context.WithTimeout(ctx, ownedStageProbeTimeout)
|
||||
defer cancel()
|
||||
@@ -438,11 +440,35 @@ func (c *Connection) ownedIdle(ctx context.Context) bool {
|
||||
var active struct {
|
||||
Data map[string]json.RawMessage `json:"data"`
|
||||
}
|
||||
return json.Unmarshal(data, &active) == nil && active.Data != nil && len(active.Data) == 0
|
||||
if json.Unmarshal(data, &active) != nil || active.Data == nil || len(active.Data) != 0 {
|
||||
return false
|
||||
}
|
||||
data, err = c.nativeRequest(probe, "get", "/api/shell", nil, "")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var shells []struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
var response struct {
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(data, &response) == nil && response.Data != nil {
|
||||
data = response.Data
|
||||
}
|
||||
if json.Unmarshal(data, &shells) != nil || shells == nil {
|
||||
return false
|
||||
}
|
||||
for _, shell := range shells {
|
||||
if shell.Status != "exited" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// StopIdleStageService stops the owned stage service on server shutdown when no
|
||||
// session is active on it. The registration (URL and password) is kept, so a
|
||||
// foreground turn or background shell is active on it. The registration (URL and password) is kept, so a
|
||||
// later start reuses the same endpoint and retained references stay valid. A
|
||||
// busy service keeps running so restart recovery can interrupt it properly.
|
||||
func StopIdleStageService() {
|
||||
|
||||
@@ -3,9 +3,12 @@ package opencode
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -63,6 +66,57 @@ func runFakeOpenCodeServe() {
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"data":{}}`))
|
||||
}))
|
||||
// A real task in this independent service keeps producing output while no
|
||||
// Pipelines adapter is connected. The marker is its explicit completion gate.
|
||||
outputPath := os.Getenv("FAKE_SHELL_OUTPUT")
|
||||
if outputPath != "" {
|
||||
go func() {
|
||||
for i := 0; ; i++ {
|
||||
if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err != nil {
|
||||
return
|
||||
}
|
||||
f, err := os.OpenFile(outputPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprintln(f, i)
|
||||
_ = f.Close()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}()
|
||||
}
|
||||
shellInfo := func() map[string]any {
|
||||
status := "exited"
|
||||
if _, err := os.Stat(os.Getenv("FAKE_SHELL_FILE")); err == nil {
|
||||
status = "running"
|
||||
}
|
||||
return map[string]any{"id": "sh_background", "status": status, "metadata": map[string]string{"sessionID": "ses_background"}, "command": "background heartbeat"}
|
||||
}
|
||||
mux.HandleFunc("/api/shell", auth(func(w http.ResponseWriter) {
|
||||
_, running := os.Stat(os.Getenv("FAKE_SHELL_FILE"))
|
||||
_, ran := os.Stat(outputPath)
|
||||
if running == nil || (outputPath != "" && ran == nil) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"data": []any{shellInfo()}})
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
}))
|
||||
mux.HandleFunc("/api/shell/sh_background", auth(func(w http.ResponseWriter) { _ = json.NewEncoder(w).Encode(map[string]any{"data": shellInfo()}) }))
|
||||
mux.HandleFunc("/api/shell/sh_background/output", func(w http.ResponseWriter, r *http.Request) {
|
||||
auth(func(w http.ResponseWriter) {
|
||||
data, _ := os.ReadFile(outputPath)
|
||||
cursor, _ := strconv.Atoi(r.URL.Query().Get("cursor"))
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if cursor > len(data) {
|
||||
cursor = len(data)
|
||||
}
|
||||
end := cursor + limit
|
||||
if end > len(data) {
|
||||
end = len(data)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"output": string(data[cursor:end]), "cursor": end, "size": len(data), "truncated": false}})
|
||||
})(w, r)
|
||||
})
|
||||
mux.HandleFunc("/api/env", auth(func(w http.ResponseWriter) {
|
||||
values := map[string]*string{}
|
||||
for _, key := range strings.Split(os.Getenv("FAKE_ENV_KEYS"), ",") {
|
||||
@@ -87,6 +141,7 @@ func fakeOwnedStageEnv(t *testing.T) string {
|
||||
t.Setenv(OpenCodeServiceStateEnvKey, t.TempDir())
|
||||
t.Setenv(fakeOpenCodeEnv, "1")
|
||||
t.Setenv("FAKE_BUSY_FILE", filepath.Join(t.TempDir(), "busy"))
|
||||
t.Setenv("FAKE_SHELL_FILE", filepath.Join(t.TempDir(), "background"))
|
||||
for _, key := range []string{StageServiceEnvKey, StageToolOutputMaxBytesEnvKey, StageToolOutputMaxLinesEnvKey, StageCompactionEnvKey, StageConfigEnvKey, "OPENCODE_CONFIG_CONTENT", "FAKE_FAIL"} {
|
||||
t.Setenv(key, "")
|
||||
}
|
||||
@@ -287,3 +342,76 @@ func waitGone(t *testing.T, pid int) {
|
||||
}
|
||||
t.Fatalf("process %d still running", pid)
|
||||
}
|
||||
|
||||
// Foreground idle does not imply service idle. This is a process-level restart:
|
||||
// the adapter closes, shutdown cleanup runs, the task keeps writing while the
|
||||
// server is away, and a new adapter reattaches the same native shell/output.
|
||||
func TestBackgroundOnlyStageSurvivesOwnerRestart(t *testing.T) {
|
||||
executable := fakeOwnedStageEnv(t)
|
||||
marker := os.Getenv("FAKE_SHELL_FILE")
|
||||
if err := os.WriteFile(marker, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := filepath.Join(t.TempDir(), "output")
|
||||
t.Setenv("FAKE_SHELL_OUTPUT", output)
|
||||
first, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pid := ownedPID(t, first)
|
||||
ref, err := first.Reference()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := NewChat(first)
|
||||
shells, err := before.Shells(context.Background(), "ses_background")
|
||||
if err != nil || len(shells) != 1 {
|
||||
t.Fatalf("native task: %+v %v", shells, err)
|
||||
}
|
||||
query := protocol.ChatShellOutputQuery{SessionID: "ses_background", ShellID: shells[0].ID, Limit: 4096}
|
||||
page, err := before.ShellOutput(context.Background(), query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.Close()
|
||||
StopIdleStageService()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
second, err := FromReference(ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer second.Close()
|
||||
if ownedPID(t, second) != pid {
|
||||
t.Fatal("restart replaced the live background service")
|
||||
}
|
||||
after := NewChat(second)
|
||||
got, err := after.Shells(context.Background(), "ses_background")
|
||||
if err != nil || len(got) != 1 || got[0].ID != shells[0].ID || got[0].Status != protocol.ChatShellRunning {
|
||||
t.Fatalf("reattached task %+v %v", got, err)
|
||||
}
|
||||
continued, err := after.ShellOutput(context.Background(), query)
|
||||
if err != nil || continued.Size <= page.Size {
|
||||
t.Fatalf("task did not progress during restart: %+v -> %+v %v", page, continued, err)
|
||||
}
|
||||
t.Logf("backend PID=%d, native shell=%s, output bytes before=%d after=%d", pid, got[0].ID, page.Size, continued.Size)
|
||||
// Configuration replacement and idle timeout must protect the same task too.
|
||||
t.Setenv(StageToolOutputMaxBytesEnvKey, "1234")
|
||||
changed, err := OpenStage(context.Background(), executable, t.TempDir(), "v2")
|
||||
if err != nil || ownedPID(t, changed) != pid {
|
||||
t.Fatalf("background service replaced on config update: %v", err)
|
||||
}
|
||||
path, _ := StageServiceRegistrationPath()
|
||||
ownedStageLocksMu.Lock()
|
||||
ownedStageUsed[path] = time.Now().Add(-time.Hour)
|
||||
ownedStageLocksMu.Unlock()
|
||||
if stopIdleOwnedStage(path, time.Minute) {
|
||||
t.Fatal("idle reaper stopped background task")
|
||||
}
|
||||
_ = os.Remove(marker)
|
||||
terminal, err := after.ShellOutput(context.Background(), query)
|
||||
if err != nil || terminal.Shell.ID != query.ShellID || terminal.Shell.Status != protocol.ChatShellExited {
|
||||
t.Fatalf("completion not observable: %+v %v", terminal, err)
|
||||
}
|
||||
StopIdleStageService()
|
||||
waitGone(t, pid)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user