Restore hosted background tasks and preserve live hosts on reconnect failure

This commit is contained in:
2026-11-07 16:22:38 +00:00
parent 5061cd5da4
commit 4ead2abdbf
4 changed files with 345 additions and 30 deletions
+71 -30
View File
@@ -97,6 +97,8 @@ func hostBinaryPath() string {
type hostLink struct {
dir string
record claudehost.Record
// Immutable capability of this attachment; old hosts cannot protect idle background tasks.
tasksKnown bool
mu sync.Mutex
conn *claudehost.Conn
@@ -134,11 +136,17 @@ type replayPlan struct {
pending map[string]bool
busyAtAcked bool
restore bool
tasks *[]claudehost.Task
}
// startHosted starts a host for args and attaches to it.
func (s *session) startHosted(ctx context.Context, args []string, mode string, env []string) (*process, error) {
c := s.chat
for _, record := range claudehost.Records(c.hostDir) {
if record.SessionID == s.id && hostReferenceLive(record) {
return nil, errors.New("the existing Claude Code host is temporarily unavailable; its work was left running, retry to reconnect")
}
}
hostConfig.Lock()
binary, hostArgs := hostConfig.binary, append([]string(nil), hostConfig.args...)
hostConfig.Unlock()
@@ -154,10 +162,10 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e
paths := claudehost.PathsFor(c.hostDir, s.id)
conn, status, err := claudehost.Attach(paths.Socket, 0, hostAttachTimeout)
if err != nil {
claudehost.KillStale(c.hostDir, claudehost.Record{SessionID: s.id, HostPID: cmd.Process.Pid})
return nil, fmt.Errorf("attach to the Claude Code host: %w", err)
log.Printf("claude chat %s: newly started host unavailable; leaving it intact for reconnect", s.id)
return nil, fmt.Errorf("attach to the Claude Code host (left running for reconnect): %w", err)
}
link := &hostLink{dir: c.hostDir, conn: conn, record: claudehost.Record{
link := &hostLink{dir: c.hostDir, conn: conn, tasksKnown: status.Tasks != nil, record: claudehost.Record{
SessionID: s.id, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: paths.Socket,
}}
p := newProcess(s, mode)
@@ -168,15 +176,14 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e
}
// connectHost attaches to a host found in the registry and prepares a
// process for s without registering it. A host that is unreachable, belongs
// to another session or never finished initializing is stopped and its files
// removed; nil is returned and the next prompt starts a fresh CLI with
// --resume.
// process for s without registering it. A live but temporarily unreachable host
// is never killed to repair the control connection. Its registration blocks a
// duplicate CLI until reconnect succeeds. Only dead references are removed.
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
if err != nil {
log.Printf("claude chat %s: host unreachable (%v); the next prompt resumes the session", record.SessionID, err)
claudehost.KillStale(c.hostDir, record)
log.Printf("claude chat %s: host unreachable (%v); retaining any live work for reconnect", record.SessionID, err)
c.removeDeadHostReference(record)
return nil, nil, replayPlan{}
}
var models []protocol.ChatModel
@@ -188,7 +195,7 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn
}
if status.SessionID != record.SessionID || err != nil {
_ = conn.Close()
claudehost.KillStale(c.hostDir, record)
c.removeDeadHostReference(record)
return nil, nil, replayPlan{}
}
directory := status.Directory
@@ -197,12 +204,12 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn
}
s := c.sessionFor(record.SessionID, directory)
p := newProcess(s, status.Mode)
p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, record: claudehost.Record{
p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, tasksKnown: status.Tasks != nil, record: claudehost.Record{
SessionID: record.SessionID, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: record.Socket,
}}
p.stderr.WriteString(status.Stderr)
p.models, p.account, p.appliedModel = models, account, status.Model
plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true}
plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true, tasks: status.Tasks}
for _, pending := range status.Pending {
plan.pending[pending.ID] = true
}
@@ -299,6 +306,14 @@ func (s *session) existingHost() *process {
func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
s := p.session
live := false
tasksReconciled := false
catchUpReady := func() {
if !tasksReconciled {
s.reconcileHostTasks(plan.tasks)
tasksReconciled = true
}
p.markReady()
}
goLive := func() {
if live {
return
@@ -336,7 +351,7 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
goLive()
}
if plan.catchUpEnd <= plan.base {
p.markReady()
catchUpReady()
}
sinceAck := 0
for {
@@ -344,9 +359,16 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
if err != nil {
goLive()
p.markReady()
if next := p.hostLost(conn, err); next != nil {
if next, status := p.hostLost(conn, err); next != nil {
conn = next
plan = replayPlan{}
plan = replayPlan{catchUpEnd: status.End, tasks: status.Tasks}
tasksReconciled = false
p.link.mu.Lock()
committed := p.link.committed
p.link.mu.Unlock()
if committed >= status.End {
catchUpReady()
}
continue
}
return
@@ -371,7 +393,7 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
goLive()
}
if frame.End >= plan.catchUpEnd {
p.markReady()
catchUpReady()
}
case claudehost.OpExit:
goLive()
@@ -416,19 +438,18 @@ func (p *process) markReady() {
// hostLost handles an ended host connection. A deliberate detach or close just
// ends the reader; an unexpected loss is retried once (the host may only have
// dropped the connection), and otherwise the host is treated as crashed: what
// it left is stopped and the turn is settled with a notice, and the next prompt
// resumes the session with --resume as after any release.
func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn {
// dropped the connection). A live host/CLI is preserved when retries fail;
// only a confirmed dead reference is removed before allowing a resumed CLI.
func (p *process) hostLost(conn *claudehost.Conn, cause error) (*claudehost.Conn, *claudehost.Status) {
link := p.link
_ = conn.Close()
if link.detached.Load() {
p.finishDetached()
return nil
return nil, nil
}
if link.closing.Load() {
p.finish(nil)
return nil
return nil, nil
}
link.mu.Lock()
committed := link.committed
@@ -438,34 +459,54 @@ func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn
if link.detached.Load() || link.closing.Load() {
break
}
next, _, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout)
next, status, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout)
if err == nil {
link.mu.Lock()
link.conn = next
link.mu.Unlock()
return next
return next, status
}
}
if link.detached.Load() {
p.finishDetached()
return nil
return nil, nil
}
if link.closing.Load() {
p.finish(nil)
return nil
return nil, nil
}
log.Printf("claude chat %s: host connection lost (%v); falling back to --resume", p.session.id, cause)
claudehost.KillStale(link.dir, link.record)
if hostReferenceLive(link.record) {
message := "Claude Code control connection is temporarily unavailable; work was left running, retry to reconnect"
log.Printf("claude chat %s: host connection lost (%v); retaining live host and CLI", p.session.id, cause)
s := p.session
s.mu.Lock()
if s.proc == p {
s.proc = nil
s.lastErr = message
}
s.mu.Unlock()
p.finishDetachedReason(message)
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventTool,
Part: &protocol.ChatPart{Type: protocol.ChatPartTool, ID: "claude-host-unavailable", Name: "Claude Code",
State: &protocol.ChatToolState{Status: protocol.ChatToolError, Error: &protocol.ChatError{Type: "connection", Message: message}}}})
return nil, nil
}
log.Printf("claude chat %s: host and CLI stopped (%v); the next prompt resumes", p.session.id, cause)
claudehost.Remove(link.dir, link.record.SessionID)
p.mu.Lock()
p.stderr.Reset()
p.mu.Unlock()
p.finish(errors.New("the Claude Code host stopped unexpectedly; the next prompt resumes this session with --resume"))
return nil
return nil, nil
}
// finishDetached ends a process this server let go of on shutdown. The CLI
// keeps running in its host; nothing is reported to watchers.
func (p *process) finishDetached() {
p.finishDetachedReason("the server is shutting down")
}
func (p *process) finishDetachedReason(reason string) {
p.mu.Lock()
if p.exited {
p.mu.Unlock()
@@ -478,7 +519,7 @@ func (p *process) finishDetached() {
p.mu.Unlock()
for _, ch := range pending {
select {
case ch <- controlReply{err: "the server is shutting down"}:
case ch <- controlReply{err: reason}:
default:
}
}
+42
View File
@@ -0,0 +1,42 @@
package claude
import (
"context"
"errors"
"fmt"
"os/exec"
"strings"
"syscall"
"time"
"git.a-wan.net/alan/pipelines/internal/claudehost"
)
// hostReferenceLive fails closed: an alive or uninspectable PID must never be
// signalled merely because its control socket did not answer. A stale reference
// to a recycled PID can delay recovery, but cannot kill unrelated work.
func hostReferenceLive(record claudehost.Record) bool {
for _, pid := range []int{record.HostPID, record.ClaudePID} {
if pid <= 1 {
continue
}
err := syscall.Kill(pid, 0)
if err != nil && !errors.Is(err, syscall.EPERM) {
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
state, inspectErr := exec.CommandContext(ctx, "ps", "-o", "stat=", "-p", fmt.Sprint(pid)).Output()
cancel()
if inspectErr == nil && strings.HasPrefix(strings.TrimSpace(string(state)), "Z") {
continue // a zombie cannot own live pipes or execute work
}
return true
}
return false
}
func (c *Chat) removeDeadHostReference(record claudehost.Record) {
if !hostReferenceLive(record) {
claudehost.Remove(c.hostDir, record.SessionID)
}
}
+70
View File
@@ -0,0 +1,70 @@
package claude
import (
"git.a-wan.net/alan/pipelines/internal/claudehost"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// reconcileHostTasks runs only after replay reaches the status snapshot's End.
// It restores native identities even when their starting records were trimmed
// from the bounded spool. Old hosts omit Tasks, which means unknown, not idle.
func (s *session) reconcileHostTasks(tasks *[]claudehost.Task) {
if tasks == nil {
return
}
s.mu.Lock()
present := make(map[string]bool, len(*tasks))
var frames []protocol.ChatShell
for _, task := range *tasks {
id := task.ToolUseID
if id == "" && task.TaskID != "" {
id = "task:" + task.TaskID
}
if id == "" {
continue
}
present[id] = true
command := boundedText(task.Command, 64<<10)
if command == "" && (task.Name == "Agent" || task.Name == "Task") && task.Description != "" {
command = "Agent: " + boundedText(task.Description, 2048)
}
s.recordShellLocked(id, command)
snapshot := s.shells[id]
snapshot.background = true
snapshot.taskID = task.TaskID
snapshot.shell.Status = protocol.ChatShellRunning
snapshot.shell.Exit = nil
if task.Description != "" {
snapshot.shell.Description = boundedText(task.Description, 2048)
}
if task.TaskType != "" {
snapshot.shell.TaskType = boundedText(task.TaskType, 128)
} else if snapshot.shell.TaskType == "" {
switch task.Name {
case "Bash":
snapshot.shell.TaskType = "shell"
case "Agent", "Task":
snapshot.shell.TaskType = "agent"
}
}
if snapshot.shell.StartedAt == 0 {
if stamp, ok := transcriptTime(task.Timestamp); ok && stamp > 0 {
snapshot.shell.StartedAt = stamp
}
}
frames = append(frames, snapshot.shell)
}
for id, snapshot := range s.shells {
if !snapshot.background || snapshot.shell.Status != protocol.ChatShellRunning || present[id] {
continue
}
// The host says this background identity is no longer running, but
// supplies neither an exit code nor a completion time. Keep both absent.
snapshot.shell.Status = protocol.ChatShellExited
frames = append(frames, snapshot.shell)
}
s.mu.Unlock()
for i := range frames {
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventShell, Shell: &frames[i]})
}
}
+162
View File
@@ -0,0 +1,162 @@
package claude
import (
"context"
"encoding/json"
"net"
"os"
"path/filepath"
"reflect"
"sync"
"syscall"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/claudehost"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestHostTaskSnapshotDistinguishesLegacyUnknownFromKnownIdle(t *testing.T) {
chat, s := backgroundSession(t)
t.Cleanup(chat.Close)
s.handleLine(nil, toolUse(t, "native-tool", "Bash", map[string]any{"command": "sleep 600", "run_in_background": true}))
s.reconcileHostTasks(nil)
if shells := running(t, s.chat); len(shells) != 1 {
t.Fatalf("legacy unknown must preserve task liveness: %+v", shells)
}
empty := []claudehost.Task{}
s.reconcileHostTasks(&empty)
if shells := running(t, s.chat); len(shells) != 0 {
t.Fatalf("authoritative idle must settle replayed background state: %+v", shells)
}
for _, source := range []string{`{}`, `{"tasks":[]}`} {
var status claudehost.Status
if err := json.Unmarshal([]byte(source), &status); err != nil {
t.Fatal(err)
}
if (status.Tasks == nil) != (source == `{}`) {
t.Fatalf("capability presence was lost: %s %+v", source, status.Tasks)
}
}
}
func TestReattachRestoresTasksAfterTheirStartWasTrimmed(t *testing.T) {
base, err := os.MkdirTemp("/tmp", "pct")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(base)
socket := filepath.Join(base, "host.sock")
listener, err := net.Listen("unix", socket)
if err != nil {
t.Fatal(err)
}
defer listener.Close()
current := []claudehost.Task{{ToolUseID: "native-bg", TaskID: "task-17", Name: "Bash", Command: "sleep 600", Description: "Wait for release", TaskType: "shell", Timestamp: "2026-10-22T12:00:00Z"}}
status := claudehost.Status{SessionID: "trimmed-session", Base: 2000, Acked: 2000, End: 2300, Init: json.RawMessage(`{"models":[{"value":"fixture-model"}]}`), Tasks: &current}
notify := make(chan struct{})
stop := make(chan struct{})
var stopOnce sync.Once
stopServer := func() { stopOnce.Do(func() { close(stop) }) }
defer stopServer()
serverDone := make(chan struct{})
go func() {
defer close(serverDone)
conn, err := listener.Accept()
if err != nil {
return
}
defer conn.Close()
var attach claudehost.Frame
if json.NewDecoder(conn).Decode(&attach) != nil {
return
}
out := json.NewEncoder(conn)
_ = out.Encode(claudehost.Frame{Op: claudehost.OpStatus, Status: &status})
// The actual running task's initiating line is older than Base. The
// retained spool contains a different, already-finished async tool.
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2000, End: 2100, Data: json.RawMessage(`{"type":"assistant","message":{"id":"historic","content":[{"type":"tool_use","id":"finished-tool","name":"Bash","input":{"command":"old job","run_in_background":true}}]}}`)})
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2100, End: 2300, Data: json.RawMessage(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"finished-tool","content":"Command running in background with ID: old"}]}}`)})
select {
case <-notify:
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","tool_use_id":"native-bg","task_id":"task-17","status":"completed","summary":"Finished"}`)})
case <-stop:
return
}
<-stop
}()
chat := newChat(context.Background())
defer chat.Close()
p, conn, plan := chat.connectHost(claudehost.Record{SessionID: status.SessionID, Socket: socket})
if p == nil {
t.Fatal("fixture host did not attach")
}
s := p.session
s.mu.Lock()
s.proc = p
s.mu.Unlock()
go p.readHost(conn, plan)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
shells, err := chat.Shells(ctx, status.SessionID)
if err != nil || len(shells) != 1 {
t.Fatalf("restore authoritative native set after replay: %+v %v", shells, err)
}
want := protocol.ChatShell{ID: shellID("native-bg"), Status: protocol.ChatShellRunning, Command: "sleep 600", Description: "Wait for release", TaskType: "shell", StartedAt: time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC).UnixMilli()}
if !reflect.DeepEqual(shells[0], want) {
t.Fatalf("native identity and metadata changed:\n got %+v\nwant %+v", shells[0], want)
}
close(notify)
eventually(t, "the restored task to finish under its original native id", func() bool {
shells, _ := chat.Shells(ctx, status.SessionID)
return len(shells) == 0
})
p.detach()
stopServer()
<-serverDone
}
func TestLiveUnresponsiveHostSurvivesAttachFailureWithoutDuplicateCLI(t *testing.T) {
fixture := hostedSetup(t)
first := hostedChat(t)
session := newCreatedChat(t, first)
frames := subscribe(t, first, session.ID)
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
record := onlyRecord(t, fixture)
first.Detach()
if err := syscall.Kill(record.HostPID, syscall.SIGSTOP); err != nil {
t.Fatal(err)
}
defer syscall.Kill(record.HostPID, syscall.SIGCONT)
after := newChat(context.Background())
defer after.Close()
if p, _, _ := after.connectHost(record); p != nil {
t.Fatal("a stopped fixture host unexpectedly answered")
}
if !alive(record.HostPID) || !alive(record.ClaudePID) {
t.Fatal("control timeout killed live work")
}
if _, err := os.Stat(claudehost.PathsFor(fixture.hostDir, record.SessionID).Record); err != nil {
t.Fatalf("live registration removed: %v", err)
}
s := after.sessionFor(session.ID, session.Directory)
if _, err := s.startHosted(context.Background(), []string{"--session-id", session.ID}, "", nil); err == nil {
t.Fatal("live unavailable session accepted a duplicate CLI start")
}
if err := syscall.Kill(record.HostPID, syscall.SIGCONT); err != nil {
t.Fatal(err)
}
if !after.attachHost(record) {
t.Fatal("same host could not reconnect after it became responsive")
}
shells, err := after.Shells(context.Background(), session.ID)
if err != nil || len(shells) != 1 || shells[0].ID != shellID("toolu_bg_fake") {
t.Fatalf("reattachment changed running task: %+v %v", shells, err)
}
if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 {
t.Fatalf("a control failure spawned a replacement CLI: %d", len(spawns))
}
}