From 4ead2abdbf49fc8942fcbd8b0f2662245ed61bfd Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 7 Nov 2026 16:22:38 +0000 Subject: [PATCH] Restore hosted background tasks and preserve live hosts on reconnect failure --- internal/claude/hosted.go | 101 ++++++++++++----- internal/claude/hosted_reference.go | 42 +++++++ internal/claude/hosted_tasks.go | 70 ++++++++++++ internal/claude/hosted_tasks_test.go | 162 +++++++++++++++++++++++++++ 4 files changed, 345 insertions(+), 30 deletions(-) create mode 100644 internal/claude/hosted_reference.go create mode 100644 internal/claude/hosted_tasks.go create mode 100644 internal/claude/hosted_tasks_test.go diff --git a/internal/claude/hosted.go b/internal/claude/hosted.go index dc42d4b7..2487d693 100644 --- a/internal/claude/hosted.go +++ b/internal/claude/hosted.go @@ -97,6 +97,8 @@ func hostBinaryPath() string { type hostLink struct { dir string record claudehost.Record + // Immutable capability of this attachment; old hosts cannot protect idle background tasks. + tasksKnown bool mu sync.Mutex conn *claudehost.Conn @@ -134,11 +136,17 @@ type replayPlan struct { pending map[string]bool busyAtAcked bool restore bool + tasks *[]claudehost.Task } // startHosted starts a host for args and attaches to it. func (s *session) startHosted(ctx context.Context, args []string, mode string, env []string) (*process, error) { c := s.chat + for _, record := range claudehost.Records(c.hostDir) { + if record.SessionID == s.id && hostReferenceLive(record) { + return nil, errors.New("the existing Claude Code host is temporarily unavailable; its work was left running, retry to reconnect") + } + } hostConfig.Lock() binary, hostArgs := hostConfig.binary, append([]string(nil), hostConfig.args...) hostConfig.Unlock() @@ -154,10 +162,10 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e paths := claudehost.PathsFor(c.hostDir, s.id) conn, status, err := claudehost.Attach(paths.Socket, 0, hostAttachTimeout) if err != nil { - claudehost.KillStale(c.hostDir, claudehost.Record{SessionID: s.id, HostPID: cmd.Process.Pid}) - return nil, fmt.Errorf("attach to the Claude Code host: %w", err) + log.Printf("claude chat %s: newly started host unavailable; leaving it intact for reconnect", s.id) + return nil, fmt.Errorf("attach to the Claude Code host (left running for reconnect): %w", err) } - link := &hostLink{dir: c.hostDir, conn: conn, record: claudehost.Record{ + link := &hostLink{dir: c.hostDir, conn: conn, tasksKnown: status.Tasks != nil, record: claudehost.Record{ SessionID: s.id, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: paths.Socket, }} p := newProcess(s, mode) @@ -168,15 +176,14 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e } // connectHost attaches to a host found in the registry and prepares a -// process for s without registering it. A host that is unreachable, belongs -// to another session or never finished initializing is stopped and its files -// removed; nil is returned and the next prompt starts a fresh CLI with -// --resume. +// process for s without registering it. A live but temporarily unreachable host +// is never killed to repair the control connection. Its registration blocks a +// duplicate CLI until reconnect succeeds. Only dead references are removed. func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) { conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout) if err != nil { - log.Printf("claude chat %s: host unreachable (%v); the next prompt resumes the session", record.SessionID, err) - claudehost.KillStale(c.hostDir, record) + log.Printf("claude chat %s: host unreachable (%v); retaining any live work for reconnect", record.SessionID, err) + c.removeDeadHostReference(record) return nil, nil, replayPlan{} } var models []protocol.ChatModel @@ -188,7 +195,7 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn } if status.SessionID != record.SessionID || err != nil { _ = conn.Close() - claudehost.KillStale(c.hostDir, record) + c.removeDeadHostReference(record) return nil, nil, replayPlan{} } directory := status.Directory @@ -197,12 +204,12 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn } s := c.sessionFor(record.SessionID, directory) p := newProcess(s, status.Mode) - p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, record: claudehost.Record{ + p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, tasksKnown: status.Tasks != nil, record: claudehost.Record{ SessionID: record.SessionID, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: record.Socket, }} p.stderr.WriteString(status.Stderr) p.models, p.account, p.appliedModel = models, account, status.Model - plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true} + plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true, tasks: status.Tasks} for _, pending := range status.Pending { plan.pending[pending.ID] = true } @@ -299,6 +306,14 @@ func (s *session) existingHost() *process { func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { s := p.session live := false + tasksReconciled := false + catchUpReady := func() { + if !tasksReconciled { + s.reconcileHostTasks(plan.tasks) + tasksReconciled = true + } + p.markReady() + } goLive := func() { if live { return @@ -336,7 +351,7 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { goLive() } if plan.catchUpEnd <= plan.base { - p.markReady() + catchUpReady() } sinceAck := 0 for { @@ -344,9 +359,16 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { if err != nil { goLive() p.markReady() - if next := p.hostLost(conn, err); next != nil { + if next, status := p.hostLost(conn, err); next != nil { conn = next - plan = replayPlan{} + plan = replayPlan{catchUpEnd: status.End, tasks: status.Tasks} + tasksReconciled = false + p.link.mu.Lock() + committed := p.link.committed + p.link.mu.Unlock() + if committed >= status.End { + catchUpReady() + } continue } return @@ -371,7 +393,7 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { goLive() } if frame.End >= plan.catchUpEnd { - p.markReady() + catchUpReady() } case claudehost.OpExit: goLive() @@ -416,19 +438,18 @@ func (p *process) markReady() { // hostLost handles an ended host connection. A deliberate detach or close just // ends the reader; an unexpected loss is retried once (the host may only have -// dropped the connection), and otherwise the host is treated as crashed: what -// it left is stopped and the turn is settled with a notice, and the next prompt -// resumes the session with --resume as after any release. -func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn { +// dropped the connection). A live host/CLI is preserved when retries fail; +// only a confirmed dead reference is removed before allowing a resumed CLI. +func (p *process) hostLost(conn *claudehost.Conn, cause error) (*claudehost.Conn, *claudehost.Status) { link := p.link _ = conn.Close() if link.detached.Load() { p.finishDetached() - return nil + return nil, nil } if link.closing.Load() { p.finish(nil) - return nil + return nil, nil } link.mu.Lock() committed := link.committed @@ -438,34 +459,54 @@ func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn if link.detached.Load() || link.closing.Load() { break } - next, _, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout) + next, status, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout) if err == nil { link.mu.Lock() link.conn = next link.mu.Unlock() - return next + return next, status } } if link.detached.Load() { p.finishDetached() - return nil + return nil, nil } if link.closing.Load() { p.finish(nil) - return nil + return nil, nil } - log.Printf("claude chat %s: host connection lost (%v); falling back to --resume", p.session.id, cause) - claudehost.KillStale(link.dir, link.record) + if hostReferenceLive(link.record) { + message := "Claude Code control connection is temporarily unavailable; work was left running, retry to reconnect" + log.Printf("claude chat %s: host connection lost (%v); retaining live host and CLI", p.session.id, cause) + s := p.session + s.mu.Lock() + if s.proc == p { + s.proc = nil + s.lastErr = message + } + s.mu.Unlock() + p.finishDetachedReason(message) + s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventTool, + Part: &protocol.ChatPart{Type: protocol.ChatPartTool, ID: "claude-host-unavailable", Name: "Claude Code", + State: &protocol.ChatToolState{Status: protocol.ChatToolError, Error: &protocol.ChatError{Type: "connection", Message: message}}}}) + return nil, nil + } + log.Printf("claude chat %s: host and CLI stopped (%v); the next prompt resumes", p.session.id, cause) + claudehost.Remove(link.dir, link.record.SessionID) p.mu.Lock() p.stderr.Reset() p.mu.Unlock() p.finish(errors.New("the Claude Code host stopped unexpectedly; the next prompt resumes this session with --resume")) - return nil + return nil, nil } // finishDetached ends a process this server let go of on shutdown. The CLI // keeps running in its host; nothing is reported to watchers. func (p *process) finishDetached() { + p.finishDetachedReason("the server is shutting down") +} + +func (p *process) finishDetachedReason(reason string) { p.mu.Lock() if p.exited { p.mu.Unlock() @@ -478,7 +519,7 @@ func (p *process) finishDetached() { p.mu.Unlock() for _, ch := range pending { select { - case ch <- controlReply{err: "the server is shutting down"}: + case ch <- controlReply{err: reason}: default: } } diff --git a/internal/claude/hosted_reference.go b/internal/claude/hosted_reference.go new file mode 100644 index 00000000..96822352 --- /dev/null +++ b/internal/claude/hosted_reference.go @@ -0,0 +1,42 @@ +package claude + +import ( + "context" + "errors" + "fmt" + "os/exec" + "strings" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" +) + +// hostReferenceLive fails closed: an alive or uninspectable PID must never be +// signalled merely because its control socket did not answer. A stale reference +// to a recycled PID can delay recovery, but cannot kill unrelated work. +func hostReferenceLive(record claudehost.Record) bool { + for _, pid := range []int{record.HostPID, record.ClaudePID} { + if pid <= 1 { + continue + } + err := syscall.Kill(pid, 0) + if err != nil && !errors.Is(err, syscall.EPERM) { + continue + } + ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + state, inspectErr := exec.CommandContext(ctx, "ps", "-o", "stat=", "-p", fmt.Sprint(pid)).Output() + cancel() + if inspectErr == nil && strings.HasPrefix(strings.TrimSpace(string(state)), "Z") { + continue // a zombie cannot own live pipes or execute work + } + return true + } + return false +} + +func (c *Chat) removeDeadHostReference(record claudehost.Record) { + if !hostReferenceLive(record) { + claudehost.Remove(c.hostDir, record.SessionID) + } +} diff --git a/internal/claude/hosted_tasks.go b/internal/claude/hosted_tasks.go new file mode 100644 index 00000000..8caf47a1 --- /dev/null +++ b/internal/claude/hosted_tasks.go @@ -0,0 +1,70 @@ +package claude + +import ( + "git.a-wan.net/alan/pipelines/internal/claudehost" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// reconcileHostTasks runs only after replay reaches the status snapshot's End. +// It restores native identities even when their starting records were trimmed +// from the bounded spool. Old hosts omit Tasks, which means unknown, not idle. +func (s *session) reconcileHostTasks(tasks *[]claudehost.Task) { + if tasks == nil { + return + } + s.mu.Lock() + present := make(map[string]bool, len(*tasks)) + var frames []protocol.ChatShell + for _, task := range *tasks { + id := task.ToolUseID + if id == "" && task.TaskID != "" { + id = "task:" + task.TaskID + } + if id == "" { + continue + } + present[id] = true + command := boundedText(task.Command, 64<<10) + if command == "" && (task.Name == "Agent" || task.Name == "Task") && task.Description != "" { + command = "Agent: " + boundedText(task.Description, 2048) + } + s.recordShellLocked(id, command) + snapshot := s.shells[id] + snapshot.background = true + snapshot.taskID = task.TaskID + snapshot.shell.Status = protocol.ChatShellRunning + snapshot.shell.Exit = nil + if task.Description != "" { + snapshot.shell.Description = boundedText(task.Description, 2048) + } + if task.TaskType != "" { + snapshot.shell.TaskType = boundedText(task.TaskType, 128) + } else if snapshot.shell.TaskType == "" { + switch task.Name { + case "Bash": + snapshot.shell.TaskType = "shell" + case "Agent", "Task": + snapshot.shell.TaskType = "agent" + } + } + if snapshot.shell.StartedAt == 0 { + if stamp, ok := transcriptTime(task.Timestamp); ok && stamp > 0 { + snapshot.shell.StartedAt = stamp + } + } + frames = append(frames, snapshot.shell) + } + for id, snapshot := range s.shells { + if !snapshot.background || snapshot.shell.Status != protocol.ChatShellRunning || present[id] { + continue + } + // The host says this background identity is no longer running, but + // supplies neither an exit code nor a completion time. Keep both absent. + snapshot.shell.Status = protocol.ChatShellExited + frames = append(frames, snapshot.shell) + } + s.mu.Unlock() + for i := range frames { + s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventShell, Shell: &frames[i]}) + } +} diff --git a/internal/claude/hosted_tasks_test.go b/internal/claude/hosted_tasks_test.go new file mode 100644 index 00000000..27bbb453 --- /dev/null +++ b/internal/claude/hosted_tasks_test.go @@ -0,0 +1,162 @@ +package claude + +import ( + "context" + "encoding/json" + "net" + "os" + "path/filepath" + "reflect" + "sync" + "syscall" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestHostTaskSnapshotDistinguishesLegacyUnknownFromKnownIdle(t *testing.T) { + chat, s := backgroundSession(t) + t.Cleanup(chat.Close) + s.handleLine(nil, toolUse(t, "native-tool", "Bash", map[string]any{"command": "sleep 600", "run_in_background": true})) + s.reconcileHostTasks(nil) + if shells := running(t, s.chat); len(shells) != 1 { + t.Fatalf("legacy unknown must preserve task liveness: %+v", shells) + } + empty := []claudehost.Task{} + s.reconcileHostTasks(&empty) + if shells := running(t, s.chat); len(shells) != 0 { + t.Fatalf("authoritative idle must settle replayed background state: %+v", shells) + } + for _, source := range []string{`{}`, `{"tasks":[]}`} { + var status claudehost.Status + if err := json.Unmarshal([]byte(source), &status); err != nil { + t.Fatal(err) + } + if (status.Tasks == nil) != (source == `{}`) { + t.Fatalf("capability presence was lost: %s %+v", source, status.Tasks) + } + } +} + +func TestReattachRestoresTasksAfterTheirStartWasTrimmed(t *testing.T) { + base, err := os.MkdirTemp("/tmp", "pct") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(base) + socket := filepath.Join(base, "host.sock") + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + current := []claudehost.Task{{ToolUseID: "native-bg", TaskID: "task-17", Name: "Bash", Command: "sleep 600", Description: "Wait for release", TaskType: "shell", Timestamp: "2026-10-22T12:00:00Z"}} + status := claudehost.Status{SessionID: "trimmed-session", Base: 2000, Acked: 2000, End: 2300, Init: json.RawMessage(`{"models":[{"value":"fixture-model"}]}`), Tasks: ¤t} + notify := make(chan struct{}) + stop := make(chan struct{}) + var stopOnce sync.Once + stopServer := func() { stopOnce.Do(func() { close(stop) }) } + defer stopServer() + serverDone := make(chan struct{}) + go func() { + defer close(serverDone) + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + var attach claudehost.Frame + if json.NewDecoder(conn).Decode(&attach) != nil { + return + } + out := json.NewEncoder(conn) + _ = out.Encode(claudehost.Frame{Op: claudehost.OpStatus, Status: &status}) + // The actual running task's initiating line is older than Base. The + // retained spool contains a different, already-finished async tool. + _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2000, End: 2100, Data: json.RawMessage(`{"type":"assistant","message":{"id":"historic","content":[{"type":"tool_use","id":"finished-tool","name":"Bash","input":{"command":"old job","run_in_background":true}}]}}`)}) + _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2100, End: 2300, Data: json.RawMessage(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"finished-tool","content":"Command running in background with ID: old"}]}}`)}) + select { + case <-notify: + _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","tool_use_id":"native-bg","task_id":"task-17","status":"completed","summary":"Finished"}`)}) + case <-stop: + return + } + <-stop + }() + chat := newChat(context.Background()) + defer chat.Close() + p, conn, plan := chat.connectHost(claudehost.Record{SessionID: status.SessionID, Socket: socket}) + if p == nil { + t.Fatal("fixture host did not attach") + } + s := p.session + s.mu.Lock() + s.proc = p + s.mu.Unlock() + go p.readHost(conn, plan) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + shells, err := chat.Shells(ctx, status.SessionID) + if err != nil || len(shells) != 1 { + t.Fatalf("restore authoritative native set after replay: %+v %v", shells, err) + } + want := protocol.ChatShell{ID: shellID("native-bg"), Status: protocol.ChatShellRunning, Command: "sleep 600", Description: "Wait for release", TaskType: "shell", StartedAt: time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC).UnixMilli()} + if !reflect.DeepEqual(shells[0], want) { + t.Fatalf("native identity and metadata changed:\n got %+v\nwant %+v", shells[0], want) + } + close(notify) + eventually(t, "the restored task to finish under its original native id", func() bool { + shells, _ := chat.Shells(ctx, status.SessionID) + return len(shells) == 0 + }) + p.detach() + stopServer() + <-serverDone +} + +func TestLiveUnresponsiveHostSurvivesAttachFailureWithoutDuplicateCLI(t *testing.T) { + fixture := hostedSetup(t) + first := hostedChat(t) + session := newCreatedChat(t, first) + frames := subscribe(t, first, session.ID) + if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + record := onlyRecord(t, fixture) + first.Detach() + if err := syscall.Kill(record.HostPID, syscall.SIGSTOP); err != nil { + t.Fatal(err) + } + defer syscall.Kill(record.HostPID, syscall.SIGCONT) + after := newChat(context.Background()) + defer after.Close() + if p, _, _ := after.connectHost(record); p != nil { + t.Fatal("a stopped fixture host unexpectedly answered") + } + if !alive(record.HostPID) || !alive(record.ClaudePID) { + t.Fatal("control timeout killed live work") + } + if _, err := os.Stat(claudehost.PathsFor(fixture.hostDir, record.SessionID).Record); err != nil { + t.Fatalf("live registration removed: %v", err) + } + s := after.sessionFor(session.ID, session.Directory) + if _, err := s.startHosted(context.Background(), []string{"--session-id", session.ID}, "", nil); err == nil { + t.Fatal("live unavailable session accepted a duplicate CLI start") + } + if err := syscall.Kill(record.HostPID, syscall.SIGCONT); err != nil { + t.Fatal(err) + } + if !after.attachHost(record) { + t.Fatal("same host could not reconnect after it became responsive") + } + shells, err := after.Shells(context.Background(), session.ID) + if err != nil || len(shells) != 1 || shells[0].ID != shellID("toolu_bg_fake") { + t.Fatalf("reattachment changed running task: %+v %v", shells, err) + } + if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 { + t.Fatalf("a control failure spawned a replacement CLI: %d", len(spawns)) + } +}