From 3da992075eb03434f2da910c8945513c9c5fa7d1 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 7 Nov 2026 16:12:52 +0000 Subject: [PATCH 1/3] Expose authoritative hosted Claude background task snapshot --- internal/claudehost/wire.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/claudehost/wire.go b/internal/claudehost/wire.go index 3482980d..e908e305 100644 --- a/internal/claudehost/wire.go +++ b/internal/claudehost/wire.go @@ -68,6 +68,19 @@ type Pending struct { Offset int64 `json:"offset"` } +// Task is a native background tool still owned by the hosted CLI. It is kept +// separately from the bounded output spool, so trimming old output cannot erase +// the identity needed by a reattaching server. Missing metadata stays absent. +type Task struct { + ToolUseID string `json:"tool_use_id,omitempty"` + TaskID string `json:"task_id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + TaskType string `json:"task_type,omitempty"` + Command string `json:"command,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + // Status is the host's snapshot of its CLI. type Status struct { Version int `json:"version"` @@ -84,6 +97,10 @@ type Status struct { // live processing resumes. Busy bool `json:"busy"` BusyAtAcked bool `json:"busy_at_acked"` + // Tasks is the authoritative running background set at End. Nil means an + // older host did not report this capability; a nonnil empty list means it + // knows that no background tasks remain. Reconcile after replay reaches End. + Tasks *[]Task `json:"tasks,omitempty"` // Init is the CLI's initialize response (models, account), so a // reconnecting server does not initialize the CLI twice. Init json.RawMessage `json:"init,omitempty"` From 5061cd5da40103e454a7ba75d7c52b6e1acc1c32 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 7 Nov 2026 16:22:16 +0000 Subject: [PATCH 2/3] Keep native Claude background tasks alive across detached sessions --- internal/claudehost/host.go | 66 ++++++---- internal/claudehost/tasks.go | 198 ++++++++++++++++++++++++++++++ internal/claudehost/tasks_test.go | 98 +++++++++++++++ 3 files changed, 339 insertions(+), 23 deletions(-) create mode 100644 internal/claudehost/tasks.go create mode 100644 internal/claudehost/tasks_test.go diff --git a/internal/claudehost/host.go b/internal/claudehost/host.go index 0235ea97..96f23eec 100644 --- a/internal/claudehost/host.go +++ b/internal/claudehost/host.go @@ -22,7 +22,7 @@ import ( // Defaults for a host started without explicit limits. const ( DefaultIdle = 20 * time.Minute - DefaultMaxDetached = 24 * time.Hour + DefaultMaxDetached = time.Duration(0) DefaultSpoolLimit = 64 << 20 maxLineBytes = 32 << 20 stderrRetained = 8 << 10 @@ -85,24 +85,26 @@ type host struct { trans []transition busy bool // pending maps unanswered CLI control request ids to their offsets. - pending map[string]int64 - initID string - init json.RawMessage - modelReqs map[string]string - modeReqs map[string]string - model string - mode string - exited bool - exitErr string - exitedAt time.Time - stderr strings.Builder - changed chan struct{} - active *client - detached time.Time - activity time.Time - stopping bool - stopped chan struct{} - claudeEnd chan struct{} + tasks map[string]string // running native background tools, independent of foreground busy + taskOrigins map[string]Task // native identity/metadata retained independently of spool compaction + pending map[string]int64 + initID string + init json.RawMessage + modelReqs map[string]string + modeReqs map[string]string + model string + mode string + exited bool + exitErr string + exitedAt time.Time + stderr strings.Builder + changed chan struct{} + active *client + detached time.Time + activity time.Time + stopping bool + stopped chan struct{} + claudeEnd chan struct{} } // Main runs a host. args are the host flags, then "--", then the Claude Code @@ -113,7 +115,7 @@ func Main(args []string) int { dir := flags.String("dir", "", "private host directory (registry, socket, spool)") id := flags.String("id", "", "Claude session id") idle := flags.Duration("idle", DefaultIdle, "release an idle CLI after this long with no server attached") - maxDetached := flags.Duration("max-detached", DefaultMaxDetached, "release the CLI after this long with no server attached, even when busy") + maxDetached := flags.Duration("max-detached", DefaultMaxDetached, "optional detached lifetime limit (0 keeps live sessions until explicitly closed)") spoolLimit := flags.Int64("spool-limit", DefaultSpoolLimit, "bytes of acknowledged output kept in the spool") if err := flags.Parse(args); err != nil { return 2 @@ -341,6 +343,7 @@ func (h *host) appendLine(line []byte) { h.end += int64(len(data)) h.lines = append(h.lines, lineRef{start: start, end: h.end}) h.activity = time.Now() + h.trackTasksLocked(line) switch env.Type { case "control_request": if env.RequestID != "" { @@ -425,10 +428,27 @@ func (h *host) statusLocked() *Status { } sort.Slice(pending, func(i, j int) bool { return pending[i].Offset < pending[j].Offset }) cwd, _ := os.Getwd() + tasks := make([]Task, 0, len(h.tasks)) + for key, taskID := range h.tasks { + task := h.taskOrigins[key] + if task.ToolUseID == "" && !strings.HasPrefix(key, "task:") { + task.ToolUseID = key + } + if task.TaskID == "" { + task.TaskID = taskID + } + tasks = append(tasks, task) + } + sort.Slice(tasks, func(i, j int) bool { + if tasks[i].ToolUseID == tasks[j].ToolUseID { + return tasks[i].TaskID < tasks[j].TaskID + } + return tasks[i].ToolUseID < tasks[j].ToolUseID + }) status := &Status{ Version: Version, SessionID: h.id, HostPID: os.Getpid(), Base: h.base, End: h.end, Acked: h.acked, Busy: h.busy, BusyAtAcked: h.busyAtLocked(h.acked), - Init: h.init, Model: h.model, Mode: h.mode, Pending: pending, + Init: h.init, Model: h.model, Mode: h.mode, Pending: pending, Tasks: &tasks, Exited: h.exited, ExitErr: h.exitErr, Stderr: h.stderr.String(), Directory: cwd, } if h.cmd != nil && h.cmd.Process != nil { @@ -688,8 +708,8 @@ func (h *host) watchIdle() { now := time.Now() h.mu.Lock() attached := h.active != nil - idle := !h.busy && len(h.pending) == 0 && now.Sub(h.detached) > h.idle && now.Sub(h.activity) > h.idle - tooLong := now.Sub(h.detached) > h.maxDetached + idle := !h.busy && len(h.pending) == 0 && len(h.tasks) == 0 && now.Sub(h.detached) > h.idle && now.Sub(h.activity) > h.idle + tooLong := h.maxDetached > 0 && now.Sub(h.detached) > h.maxDetached exitedLong := h.exited && now.Sub(h.exitedAt) > h.idle h.mu.Unlock() if attached { diff --git a/internal/claudehost/tasks.go b/internal/claudehost/tasks.go new file mode 100644 index 00000000..fd24abcf --- /dev/null +++ b/internal/claudehost/tasks.go @@ -0,0 +1,198 @@ +package claudehost + +import ( + "encoding/json" + "regexp" + "strings" + "unicode/utf8" +) + +// The host owns the real CLI even with no Pipelines connection. Foreground +// result messages do not end background tools; only their native completion +// notifications do. Keep this lifecycle state independently of spool acks. +func (h *host) trackTasksLocked(line []byte) { + var event struct { + Type string `json:"type"` + Subtype string `json:"subtype"` + ToolUseID string `json:"tool_use_id"` + TaskID string `json:"task_id"` + Status string `json:"status"` + Description string `json:"description"` + TaskType string `json:"task_type"` + Timestamp string `json:"timestamp"` + Message struct { + Content json.RawMessage `json:"content"` + } `json:"message"` + } + if json.Unmarshal(line, &event) != nil { + return + } + if h.tasks == nil { + h.tasks = map[string]string{} + } + if h.taskOrigins == nil { + h.taskOrigins = map[string]Task{} + } + switch event.Subtype { + case "task_started", "task_progress": + id := event.ToolUseID + if id == "" { + for key, task := range h.tasks { + if event.TaskID != "" && task == event.TaskID { + id = key + break + } + } + if id == "" { + id = "task:" + event.TaskID + } + } + if id != "task:" { + if event.ToolUseID != "" && event.TaskID != "" { + fallback := "task:" + event.TaskID + if prior, exists := h.taskOrigins[fallback]; exists { + if _, recorded := h.taskOrigins[id]; !recorded { + h.taskOrigins[id] = prior + } + delete(h.tasks, fallback) + delete(h.taskOrigins, fallback) + } + } + h.tasks[id] = event.TaskID + origin := h.taskOrigins[id] + if event.ToolUseID != "" { + origin.ToolUseID = event.ToolUseID + } + if event.TaskID != "" { + origin.TaskID = event.TaskID + } + if event.Description != "" { + origin.Description = taskText(event.Description, 2048) + } + if event.TaskType != "" { + origin.TaskType = taskText(event.TaskType, 128) + } + if origin.Timestamp == "" { + origin.Timestamp = event.Timestamp + } + h.taskOrigins[id] = origin + } + case "task_notification": + if event.Status == "completed" || event.Status == "failed" || event.Status == "stopped" { + for key, task := range h.tasks { + if key == event.ToolUseID || (event.TaskID != "" && (task == event.TaskID || key == "task:"+event.TaskID)) { + delete(h.tasks, key) + delete(h.taskOrigins, key) + } + } + } + } + if event.Type == "result" { + for key := range h.taskOrigins { + if _, running := h.tasks[key]; !running { + delete(h.taskOrigins, key) + } + } + } + var blocks []taskBlock + if json.Unmarshal(event.Message.Content, &blocks) != nil { + var text string + if json.Unmarshal(event.Message.Content, &text) != nil { + return + } + blocks = []taskBlock{{Type: "text", Text: text}} + } + + finishText := func(text string) { + for _, match := range completedTaskNotice.FindAllStringSubmatch(text, -1) { + key := match[1] + taskID := h.tasks[key] + delete(h.tasks, key) + delete(h.taskOrigins, key) + if taskID != "" { + delete(h.tasks, "task:"+taskID) + delete(h.taskOrigins, "task:"+taskID) + } + } + } + for _, block := range blocks { + switch block.Type { + case "tool_use": + if block.ID != "" && (block.Name == "Agent" || block.Name == "Task" || block.Name == "Bash") { + if block.Input.Background { + if _, exists := h.tasks[block.ID]; !exists { + h.tasks[block.ID] = "" + } + } + origin := h.taskOrigins[block.ID] + origin.ToolUseID, origin.Name = block.ID, block.Name + origin.Command = taskText(block.Input.Command, 64<<10) + if block.Input.Description != "" { + origin.Description = taskText(block.Input.Description, 2048) + } + if origin.TaskType == "" { + if block.Name == "Bash" { + origin.TaskType = "shell" + } else { + origin.TaskType = "agent" + } + } + if origin.Timestamp == "" { + origin.Timestamp = event.Timestamp + } + h.taskOrigins[block.ID] = origin + } + case "tool_result": + var text string + if json.Unmarshal(block.Content, &text) != nil { + var parts []struct { + Text string `json:"text"` + } + _ = json.Unmarshal(block.Content, &parts) + for _, part := range parts { + text += part.Text + "\n" + } + } + if strings.Contains(text, "running in background with ID") || strings.HasPrefix(strings.TrimSpace(text), "Async agent launched") { + if block.ToolUseID != "" { + if _, exists := h.tasks[block.ToolUseID]; !exists { + h.tasks[block.ToolUseID] = "" + } + } + } else { + delete(h.tasks, block.ToolUseID) + delete(h.taskOrigins, block.ToolUseID) + } + finishText(text) + case "text": + finishText(block.Text) + } + } +} + +var completedTaskNotice = regexp.MustCompile(`(?s).*?\s*([^<\s]+)\s*.*?`) + +// Native stream-json message content can be an array or a plain notification. +type taskBlock struct { + Type string `json:"type"` + ID string `json:"id"` + ToolUseID string `json:"tool_use_id"` + Name string `json:"name"` + Input struct { + Background bool `json:"run_in_background"` + Command string `json:"command"` + Description string `json:"description"` + } `json:"input"` + Text string `json:"text"` + Content json.RawMessage `json:"content"` +} + +func taskText(s string, limit int) string { + if len(s) <= limit { + return s + } + for limit > 0 && !utf8.ValidString(s[:limit]) { + limit-- + } + return s[:limit] +} diff --git a/internal/claudehost/tasks_test.go b/internal/claudehost/tasks_test.go new file mode 100644 index 00000000..693122e5 --- /dev/null +++ b/internal/claudehost/tasks_test.go @@ -0,0 +1,98 @@ +package claudehost + +import ( + "os" + "syscall" + "testing" + "time" +) + +func TestNativeTaskSnapshotOutlivesForegroundAndSpool(t *testing.T) { + h := &host{} + h.trackTasksLocked([]byte(`{"type":"assistant","timestamp":"2026-11-07T12:00:00Z","message":{"content":[{"type":"tool_use","id":"tool-bg","name":"Bash","input":{"command":"sleep 600","run_in_background":true}}]}}`)) + h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_started","tool_use_id":"tool-bg","task_id":"native-bg","description":"Background build","task_type":"local_bash"}`)) + h.trackTasksLocked([]byte(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool-bg","content":"Command running in background with ID: native-bg"}]}}`)) + h.trackTasksLocked([]byte(`{"type":"result"}`)) + h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_progress","task_id":"native-bg"}`)) + // Acknowledged stream data may be compacted; live membership must not be. + h.base, h.end, h.acked = 10000, 10000, 10000 + h.lines = nil + status := h.statusLocked() + if status.Tasks == nil || len(*status.Tasks) != 1 { + t.Fatalf("native snapshot: %+v", status.Tasks) + } + task := (*status.Tasks)[0] + if task.ToolUseID != "tool-bg" || task.TaskID != "native-bg" || task.Command != "sleep 600" || task.Description != "Background build" || task.Timestamp != "2026-11-07T12:00:00Z" { + t.Fatalf("identity/metadata lost: %+v", task) + } + h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_notification","task_id":"native-bg","status":"completed"}`)) + if status = h.statusLocked(); status.Tasks == nil || len(*status.Tasks) != 0 { + t.Fatalf("completion not authoritative: %+v", status.Tasks) + } +} + +func TestNativeTaskAliasesAndTextCompletion(t *testing.T) { + h := &host{} + h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_progress","task_id":"native-agent"}`)) + h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_started","task_id":"native-agent","tool_use_id":"tool-agent","description":"Review"}`)) + h.trackTasksLocked([]byte(`{"type":"assistant","message":{"content":[{"type":"tool_use","id":"tool-agent","name":"Agent","input":{"description":"Review"}}]}}`)) + h.trackTasksLocked([]byte(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool-agent","content":[{"type":"text","text":"Async agent launched successfully."}]}]}}`)) + if len(h.tasks) != 1 || h.tasks["tool-agent"] != "native-agent" { + t.Fatalf("alias lost or duplicated: %+v", h.tasks) + } + h.trackTasksLocked([]byte(`{"type":"user","message":{"content":"tool-agentcompleted"}}`)) + if len(h.tasks) != 0 || len(h.taskOrigins) != 0 { + t.Fatalf("text completion retained native work: %+v %+v", h.tasks, h.taskOrigins) + } + h.trackTasksLocked([]byte(`{"type":"assistant","message":{"content":[{"type":"tool_use","id":"foreground","name":"Agent","input":{"description":"Sync review"}}]}}`)) + h.trackTasksLocked([]byte(`{"type":"result"}`)) + if len(h.tasks) != 0 || len(h.taskOrigins) != 0 { + t.Fatal("foreground task leaked into detached liveness") + } +} + +func TestDetachedBackgroundCLIIsNotReapedAsIdle(t *testing.T) { + dir, err := os.MkdirTemp("/tmp", "phbg") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + now := time.Now() + h := &host{dir: dir, id: "background-survival", paths: PathsFor(dir, "background-survival"), idle: 50 * time.Millisecond, maxDetached: DefaultMaxDetached, spoolLimit: 64 << 20, pending: map[string]int64{}, modelReqs: map[string]string{}, modeReqs: map[string]string{}, changed: make(chan struct{}), stopped: make(chan struct{}), claudeEnd: make(chan struct{}), activity: now, detached: now} + // This is an actual CLI process owned by a separate host lifecycle, not a + // synthetic task row. It remains quiet after finishing its foreground turn. + script := `printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"tool_use","id":"tool-bg","name":"Bash","input":{"command":"quiet work","run_in_background":true}}]}}' '{"type":"result"}'; sleep 30` + if err := h.start([]string{"/bin/sh", "-c", script}); err != nil { + t.Fatal(err) + } + defer h.shutdown("test cleanup") + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + h.mu.Lock() + ready := len(h.tasks) == 1 && !h.busy + h.mu.Unlock() + if ready { + break + } + time.Sleep(time.Millisecond) + } + go h.watchIdle() + time.Sleep(180 * time.Millisecond) + select { + case <-h.stopped: + t.Fatal("detached background-only CLI was killed by idle expiry") + default: + } + if syscall.Kill(h.cmd.Process.Pid, 0) != nil { + t.Fatal("native CLI did not survive detached idle window") + } + t.Logf("native CLI PID=%d survived detached background-only idle window", h.cmd.Process.Pid) + h.mu.Lock() + h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_notification","tool_use_id":"tool-bg","status":"completed"}`)) + h.mu.Unlock() + select { + case <-h.stopped: + case <-time.After(5 * time.Second): + t.Fatal("completed idle CLI was not released") + } +} From 4ead2abdbf49fc8942fcbd8b0f2662245ed61bfd Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 7 Nov 2026 16:22:38 +0000 Subject: [PATCH 3/3] Restore hosted background tasks and preserve live hosts on reconnect failure --- internal/claude/hosted.go | 101 ++++++++++++----- internal/claude/hosted_reference.go | 42 +++++++ internal/claude/hosted_tasks.go | 70 ++++++++++++ internal/claude/hosted_tasks_test.go | 162 +++++++++++++++++++++++++++ 4 files changed, 345 insertions(+), 30 deletions(-) create mode 100644 internal/claude/hosted_reference.go create mode 100644 internal/claude/hosted_tasks.go create mode 100644 internal/claude/hosted_tasks_test.go diff --git a/internal/claude/hosted.go b/internal/claude/hosted.go index dc42d4b7..2487d693 100644 --- a/internal/claude/hosted.go +++ b/internal/claude/hosted.go @@ -97,6 +97,8 @@ func hostBinaryPath() string { type hostLink struct { dir string record claudehost.Record + // Immutable capability of this attachment; old hosts cannot protect idle background tasks. + tasksKnown bool mu sync.Mutex conn *claudehost.Conn @@ -134,11 +136,17 @@ type replayPlan struct { pending map[string]bool busyAtAcked bool restore bool + tasks *[]claudehost.Task } // startHosted starts a host for args and attaches to it. func (s *session) startHosted(ctx context.Context, args []string, mode string, env []string) (*process, error) { c := s.chat + for _, record := range claudehost.Records(c.hostDir) { + if record.SessionID == s.id && hostReferenceLive(record) { + return nil, errors.New("the existing Claude Code host is temporarily unavailable; its work was left running, retry to reconnect") + } + } hostConfig.Lock() binary, hostArgs := hostConfig.binary, append([]string(nil), hostConfig.args...) hostConfig.Unlock() @@ -154,10 +162,10 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e paths := claudehost.PathsFor(c.hostDir, s.id) conn, status, err := claudehost.Attach(paths.Socket, 0, hostAttachTimeout) if err != nil { - claudehost.KillStale(c.hostDir, claudehost.Record{SessionID: s.id, HostPID: cmd.Process.Pid}) - return nil, fmt.Errorf("attach to the Claude Code host: %w", err) + log.Printf("claude chat %s: newly started host unavailable; leaving it intact for reconnect", s.id) + return nil, fmt.Errorf("attach to the Claude Code host (left running for reconnect): %w", err) } - link := &hostLink{dir: c.hostDir, conn: conn, record: claudehost.Record{ + link := &hostLink{dir: c.hostDir, conn: conn, tasksKnown: status.Tasks != nil, record: claudehost.Record{ SessionID: s.id, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: paths.Socket, }} p := newProcess(s, mode) @@ -168,15 +176,14 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e } // connectHost attaches to a host found in the registry and prepares a -// process for s without registering it. A host that is unreachable, belongs -// to another session or never finished initializing is stopped and its files -// removed; nil is returned and the next prompt starts a fresh CLI with -// --resume. +// process for s without registering it. A live but temporarily unreachable host +// is never killed to repair the control connection. Its registration blocks a +// duplicate CLI until reconnect succeeds. Only dead references are removed. func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) { conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout) if err != nil { - log.Printf("claude chat %s: host unreachable (%v); the next prompt resumes the session", record.SessionID, err) - claudehost.KillStale(c.hostDir, record) + log.Printf("claude chat %s: host unreachable (%v); retaining any live work for reconnect", record.SessionID, err) + c.removeDeadHostReference(record) return nil, nil, replayPlan{} } var models []protocol.ChatModel @@ -188,7 +195,7 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn } if status.SessionID != record.SessionID || err != nil { _ = conn.Close() - claudehost.KillStale(c.hostDir, record) + c.removeDeadHostReference(record) return nil, nil, replayPlan{} } directory := status.Directory @@ -197,12 +204,12 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn } s := c.sessionFor(record.SessionID, directory) p := newProcess(s, status.Mode) - p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, record: claudehost.Record{ + p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, tasksKnown: status.Tasks != nil, record: claudehost.Record{ SessionID: record.SessionID, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: record.Socket, }} p.stderr.WriteString(status.Stderr) p.models, p.account, p.appliedModel = models, account, status.Model - plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true} + plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true, tasks: status.Tasks} for _, pending := range status.Pending { plan.pending[pending.ID] = true } @@ -299,6 +306,14 @@ func (s *session) existingHost() *process { func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { s := p.session live := false + tasksReconciled := false + catchUpReady := func() { + if !tasksReconciled { + s.reconcileHostTasks(plan.tasks) + tasksReconciled = true + } + p.markReady() + } goLive := func() { if live { return @@ -336,7 +351,7 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { goLive() } if plan.catchUpEnd <= plan.base { - p.markReady() + catchUpReady() } sinceAck := 0 for { @@ -344,9 +359,16 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { if err != nil { goLive() p.markReady() - if next := p.hostLost(conn, err); next != nil { + if next, status := p.hostLost(conn, err); next != nil { conn = next - plan = replayPlan{} + plan = replayPlan{catchUpEnd: status.End, tasks: status.Tasks} + tasksReconciled = false + p.link.mu.Lock() + committed := p.link.committed + p.link.mu.Unlock() + if committed >= status.End { + catchUpReady() + } continue } return @@ -371,7 +393,7 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { goLive() } if frame.End >= plan.catchUpEnd { - p.markReady() + catchUpReady() } case claudehost.OpExit: goLive() @@ -416,19 +438,18 @@ func (p *process) markReady() { // hostLost handles an ended host connection. A deliberate detach or close just // ends the reader; an unexpected loss is retried once (the host may only have -// dropped the connection), and otherwise the host is treated as crashed: what -// it left is stopped and the turn is settled with a notice, and the next prompt -// resumes the session with --resume as after any release. -func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn { +// dropped the connection). A live host/CLI is preserved when retries fail; +// only a confirmed dead reference is removed before allowing a resumed CLI. +func (p *process) hostLost(conn *claudehost.Conn, cause error) (*claudehost.Conn, *claudehost.Status) { link := p.link _ = conn.Close() if link.detached.Load() { p.finishDetached() - return nil + return nil, nil } if link.closing.Load() { p.finish(nil) - return nil + return nil, nil } link.mu.Lock() committed := link.committed @@ -438,34 +459,54 @@ func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn if link.detached.Load() || link.closing.Load() { break } - next, _, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout) + next, status, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout) if err == nil { link.mu.Lock() link.conn = next link.mu.Unlock() - return next + return next, status } } if link.detached.Load() { p.finishDetached() - return nil + return nil, nil } if link.closing.Load() { p.finish(nil) - return nil + return nil, nil } - log.Printf("claude chat %s: host connection lost (%v); falling back to --resume", p.session.id, cause) - claudehost.KillStale(link.dir, link.record) + if hostReferenceLive(link.record) { + message := "Claude Code control connection is temporarily unavailable; work was left running, retry to reconnect" + log.Printf("claude chat %s: host connection lost (%v); retaining live host and CLI", p.session.id, cause) + s := p.session + s.mu.Lock() + if s.proc == p { + s.proc = nil + s.lastErr = message + } + s.mu.Unlock() + p.finishDetachedReason(message) + s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventTool, + Part: &protocol.ChatPart{Type: protocol.ChatPartTool, ID: "claude-host-unavailable", Name: "Claude Code", + State: &protocol.ChatToolState{Status: protocol.ChatToolError, Error: &protocol.ChatError{Type: "connection", Message: message}}}}) + return nil, nil + } + log.Printf("claude chat %s: host and CLI stopped (%v); the next prompt resumes", p.session.id, cause) + claudehost.Remove(link.dir, link.record.SessionID) p.mu.Lock() p.stderr.Reset() p.mu.Unlock() p.finish(errors.New("the Claude Code host stopped unexpectedly; the next prompt resumes this session with --resume")) - return nil + return nil, nil } // finishDetached ends a process this server let go of on shutdown. The CLI // keeps running in its host; nothing is reported to watchers. func (p *process) finishDetached() { + p.finishDetachedReason("the server is shutting down") +} + +func (p *process) finishDetachedReason(reason string) { p.mu.Lock() if p.exited { p.mu.Unlock() @@ -478,7 +519,7 @@ func (p *process) finishDetached() { p.mu.Unlock() for _, ch := range pending { select { - case ch <- controlReply{err: "the server is shutting down"}: + case ch <- controlReply{err: reason}: default: } } diff --git a/internal/claude/hosted_reference.go b/internal/claude/hosted_reference.go new file mode 100644 index 00000000..96822352 --- /dev/null +++ b/internal/claude/hosted_reference.go @@ -0,0 +1,42 @@ +package claude + +import ( + "context" + "errors" + "fmt" + "os/exec" + "strings" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" +) + +// hostReferenceLive fails closed: an alive or uninspectable PID must never be +// signalled merely because its control socket did not answer. A stale reference +// to a recycled PID can delay recovery, but cannot kill unrelated work. +func hostReferenceLive(record claudehost.Record) bool { + for _, pid := range []int{record.HostPID, record.ClaudePID} { + if pid <= 1 { + continue + } + err := syscall.Kill(pid, 0) + if err != nil && !errors.Is(err, syscall.EPERM) { + continue + } + ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + state, inspectErr := exec.CommandContext(ctx, "ps", "-o", "stat=", "-p", fmt.Sprint(pid)).Output() + cancel() + if inspectErr == nil && strings.HasPrefix(strings.TrimSpace(string(state)), "Z") { + continue // a zombie cannot own live pipes or execute work + } + return true + } + return false +} + +func (c *Chat) removeDeadHostReference(record claudehost.Record) { + if !hostReferenceLive(record) { + claudehost.Remove(c.hostDir, record.SessionID) + } +} diff --git a/internal/claude/hosted_tasks.go b/internal/claude/hosted_tasks.go new file mode 100644 index 00000000..8caf47a1 --- /dev/null +++ b/internal/claude/hosted_tasks.go @@ -0,0 +1,70 @@ +package claude + +import ( + "git.a-wan.net/alan/pipelines/internal/claudehost" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// reconcileHostTasks runs only after replay reaches the status snapshot's End. +// It restores native identities even when their starting records were trimmed +// from the bounded spool. Old hosts omit Tasks, which means unknown, not idle. +func (s *session) reconcileHostTasks(tasks *[]claudehost.Task) { + if tasks == nil { + return + } + s.mu.Lock() + present := make(map[string]bool, len(*tasks)) + var frames []protocol.ChatShell + for _, task := range *tasks { + id := task.ToolUseID + if id == "" && task.TaskID != "" { + id = "task:" + task.TaskID + } + if id == "" { + continue + } + present[id] = true + command := boundedText(task.Command, 64<<10) + if command == "" && (task.Name == "Agent" || task.Name == "Task") && task.Description != "" { + command = "Agent: " + boundedText(task.Description, 2048) + } + s.recordShellLocked(id, command) + snapshot := s.shells[id] + snapshot.background = true + snapshot.taskID = task.TaskID + snapshot.shell.Status = protocol.ChatShellRunning + snapshot.shell.Exit = nil + if task.Description != "" { + snapshot.shell.Description = boundedText(task.Description, 2048) + } + if task.TaskType != "" { + snapshot.shell.TaskType = boundedText(task.TaskType, 128) + } else if snapshot.shell.TaskType == "" { + switch task.Name { + case "Bash": + snapshot.shell.TaskType = "shell" + case "Agent", "Task": + snapshot.shell.TaskType = "agent" + } + } + if snapshot.shell.StartedAt == 0 { + if stamp, ok := transcriptTime(task.Timestamp); ok && stamp > 0 { + snapshot.shell.StartedAt = stamp + } + } + frames = append(frames, snapshot.shell) + } + for id, snapshot := range s.shells { + if !snapshot.background || snapshot.shell.Status != protocol.ChatShellRunning || present[id] { + continue + } + // The host says this background identity is no longer running, but + // supplies neither an exit code nor a completion time. Keep both absent. + snapshot.shell.Status = protocol.ChatShellExited + frames = append(frames, snapshot.shell) + } + s.mu.Unlock() + for i := range frames { + s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventShell, Shell: &frames[i]}) + } +} diff --git a/internal/claude/hosted_tasks_test.go b/internal/claude/hosted_tasks_test.go new file mode 100644 index 00000000..27bbb453 --- /dev/null +++ b/internal/claude/hosted_tasks_test.go @@ -0,0 +1,162 @@ +package claude + +import ( + "context" + "encoding/json" + "net" + "os" + "path/filepath" + "reflect" + "sync" + "syscall" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestHostTaskSnapshotDistinguishesLegacyUnknownFromKnownIdle(t *testing.T) { + chat, s := backgroundSession(t) + t.Cleanup(chat.Close) + s.handleLine(nil, toolUse(t, "native-tool", "Bash", map[string]any{"command": "sleep 600", "run_in_background": true})) + s.reconcileHostTasks(nil) + if shells := running(t, s.chat); len(shells) != 1 { + t.Fatalf("legacy unknown must preserve task liveness: %+v", shells) + } + empty := []claudehost.Task{} + s.reconcileHostTasks(&empty) + if shells := running(t, s.chat); len(shells) != 0 { + t.Fatalf("authoritative idle must settle replayed background state: %+v", shells) + } + for _, source := range []string{`{}`, `{"tasks":[]}`} { + var status claudehost.Status + if err := json.Unmarshal([]byte(source), &status); err != nil { + t.Fatal(err) + } + if (status.Tasks == nil) != (source == `{}`) { + t.Fatalf("capability presence was lost: %s %+v", source, status.Tasks) + } + } +} + +func TestReattachRestoresTasksAfterTheirStartWasTrimmed(t *testing.T) { + base, err := os.MkdirTemp("/tmp", "pct") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(base) + socket := filepath.Join(base, "host.sock") + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + current := []claudehost.Task{{ToolUseID: "native-bg", TaskID: "task-17", Name: "Bash", Command: "sleep 600", Description: "Wait for release", TaskType: "shell", Timestamp: "2026-10-22T12:00:00Z"}} + status := claudehost.Status{SessionID: "trimmed-session", Base: 2000, Acked: 2000, End: 2300, Init: json.RawMessage(`{"models":[{"value":"fixture-model"}]}`), Tasks: ¤t} + notify := make(chan struct{}) + stop := make(chan struct{}) + var stopOnce sync.Once + stopServer := func() { stopOnce.Do(func() { close(stop) }) } + defer stopServer() + serverDone := make(chan struct{}) + go func() { + defer close(serverDone) + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + var attach claudehost.Frame + if json.NewDecoder(conn).Decode(&attach) != nil { + return + } + out := json.NewEncoder(conn) + _ = out.Encode(claudehost.Frame{Op: claudehost.OpStatus, Status: &status}) + // The actual running task's initiating line is older than Base. The + // retained spool contains a different, already-finished async tool. + _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2000, End: 2100, Data: json.RawMessage(`{"type":"assistant","message":{"id":"historic","content":[{"type":"tool_use","id":"finished-tool","name":"Bash","input":{"command":"old job","run_in_background":true}}]}}`)}) + _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2100, End: 2300, Data: json.RawMessage(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"finished-tool","content":"Command running in background with ID: old"}]}}`)}) + select { + case <-notify: + _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","tool_use_id":"native-bg","task_id":"task-17","status":"completed","summary":"Finished"}`)}) + case <-stop: + return + } + <-stop + }() + chat := newChat(context.Background()) + defer chat.Close() + p, conn, plan := chat.connectHost(claudehost.Record{SessionID: status.SessionID, Socket: socket}) + if p == nil { + t.Fatal("fixture host did not attach") + } + s := p.session + s.mu.Lock() + s.proc = p + s.mu.Unlock() + go p.readHost(conn, plan) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + shells, err := chat.Shells(ctx, status.SessionID) + if err != nil || len(shells) != 1 { + t.Fatalf("restore authoritative native set after replay: %+v %v", shells, err) + } + want := protocol.ChatShell{ID: shellID("native-bg"), Status: protocol.ChatShellRunning, Command: "sleep 600", Description: "Wait for release", TaskType: "shell", StartedAt: time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC).UnixMilli()} + if !reflect.DeepEqual(shells[0], want) { + t.Fatalf("native identity and metadata changed:\n got %+v\nwant %+v", shells[0], want) + } + close(notify) + eventually(t, "the restored task to finish under its original native id", func() bool { + shells, _ := chat.Shells(ctx, status.SessionID) + return len(shells) == 0 + }) + p.detach() + stopServer() + <-serverDone +} + +func TestLiveUnresponsiveHostSurvivesAttachFailureWithoutDuplicateCLI(t *testing.T) { + fixture := hostedSetup(t) + first := hostedChat(t) + session := newCreatedChat(t, first) + frames := subscribe(t, first, session.ID) + if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + record := onlyRecord(t, fixture) + first.Detach() + if err := syscall.Kill(record.HostPID, syscall.SIGSTOP); err != nil { + t.Fatal(err) + } + defer syscall.Kill(record.HostPID, syscall.SIGCONT) + after := newChat(context.Background()) + defer after.Close() + if p, _, _ := after.connectHost(record); p != nil { + t.Fatal("a stopped fixture host unexpectedly answered") + } + if !alive(record.HostPID) || !alive(record.ClaudePID) { + t.Fatal("control timeout killed live work") + } + if _, err := os.Stat(claudehost.PathsFor(fixture.hostDir, record.SessionID).Record); err != nil { + t.Fatalf("live registration removed: %v", err) + } + s := after.sessionFor(session.ID, session.Directory) + if _, err := s.startHosted(context.Background(), []string{"--session-id", session.ID}, "", nil); err == nil { + t.Fatal("live unavailable session accepted a duplicate CLI start") + } + if err := syscall.Kill(record.HostPID, syscall.SIGCONT); err != nil { + t.Fatal(err) + } + if !after.attachHost(record) { + t.Fatal("same host could not reconnect after it became responsive") + } + shells, err := after.Shells(context.Background(), session.ID) + if err != nil || len(shells) != 1 || shells[0].ID != shellID("toolu_bg_fake") { + t.Fatalf("reattachment changed running task: %+v %v", shells, err) + } + if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 { + t.Fatalf("a control failure spawned a replacement CLI: %d", len(spawns)) + } +}