Merge durable native host task tracking
This commit is contained in:
+71
-30
@@ -97,6 +97,8 @@ func hostBinaryPath() string {
|
||||
type hostLink struct {
|
||||
dir string
|
||||
record claudehost.Record
|
||||
// Immutable capability of this attachment; old hosts cannot protect idle background tasks.
|
||||
tasksKnown bool
|
||||
|
||||
mu sync.Mutex
|
||||
conn *claudehost.Conn
|
||||
@@ -134,11 +136,17 @@ type replayPlan struct {
|
||||
pending map[string]bool
|
||||
busyAtAcked bool
|
||||
restore bool
|
||||
tasks *[]claudehost.Task
|
||||
}
|
||||
|
||||
// startHosted starts a host for args and attaches to it.
|
||||
func (s *session) startHosted(ctx context.Context, args []string, mode string, env []string) (*process, error) {
|
||||
c := s.chat
|
||||
for _, record := range claudehost.Records(c.hostDir) {
|
||||
if record.SessionID == s.id && hostReferenceLive(record) {
|
||||
return nil, errors.New("the existing Claude Code host is temporarily unavailable; its work was left running, retry to reconnect")
|
||||
}
|
||||
}
|
||||
hostConfig.Lock()
|
||||
binary, hostArgs := hostConfig.binary, append([]string(nil), hostConfig.args...)
|
||||
hostConfig.Unlock()
|
||||
@@ -154,10 +162,10 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e
|
||||
paths := claudehost.PathsFor(c.hostDir, s.id)
|
||||
conn, status, err := claudehost.Attach(paths.Socket, 0, hostAttachTimeout)
|
||||
if err != nil {
|
||||
claudehost.KillStale(c.hostDir, claudehost.Record{SessionID: s.id, HostPID: cmd.Process.Pid})
|
||||
return nil, fmt.Errorf("attach to the Claude Code host: %w", err)
|
||||
log.Printf("claude chat %s: newly started host unavailable; leaving it intact for reconnect", s.id)
|
||||
return nil, fmt.Errorf("attach to the Claude Code host (left running for reconnect): %w", err)
|
||||
}
|
||||
link := &hostLink{dir: c.hostDir, conn: conn, record: claudehost.Record{
|
||||
link := &hostLink{dir: c.hostDir, conn: conn, tasksKnown: status.Tasks != nil, record: claudehost.Record{
|
||||
SessionID: s.id, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: paths.Socket,
|
||||
}}
|
||||
p := newProcess(s, mode)
|
||||
@@ -168,15 +176,14 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e
|
||||
}
|
||||
|
||||
// connectHost attaches to a host found in the registry and prepares a
|
||||
// process for s without registering it. A host that is unreachable, belongs
|
||||
// to another session or never finished initializing is stopped and its files
|
||||
// removed; nil is returned and the next prompt starts a fresh CLI with
|
||||
// --resume.
|
||||
// process for s without registering it. A live but temporarily unreachable host
|
||||
// is never killed to repair the control connection. Its registration blocks a
|
||||
// duplicate CLI until reconnect succeeds. Only dead references are removed.
|
||||
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
|
||||
conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
|
||||
if err != nil {
|
||||
log.Printf("claude chat %s: host unreachable (%v); the next prompt resumes the session", record.SessionID, err)
|
||||
claudehost.KillStale(c.hostDir, record)
|
||||
log.Printf("claude chat %s: host unreachable (%v); retaining any live work for reconnect", record.SessionID, err)
|
||||
c.removeDeadHostReference(record)
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
var models []protocol.ChatModel
|
||||
@@ -188,7 +195,7 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn
|
||||
}
|
||||
if status.SessionID != record.SessionID || err != nil {
|
||||
_ = conn.Close()
|
||||
claudehost.KillStale(c.hostDir, record)
|
||||
c.removeDeadHostReference(record)
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
directory := status.Directory
|
||||
@@ -197,12 +204,12 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn
|
||||
}
|
||||
s := c.sessionFor(record.SessionID, directory)
|
||||
p := newProcess(s, status.Mode)
|
||||
p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, record: claudehost.Record{
|
||||
p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, tasksKnown: status.Tasks != nil, record: claudehost.Record{
|
||||
SessionID: record.SessionID, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: record.Socket,
|
||||
}}
|
||||
p.stderr.WriteString(status.Stderr)
|
||||
p.models, p.account, p.appliedModel = models, account, status.Model
|
||||
plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true}
|
||||
plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true, tasks: status.Tasks}
|
||||
for _, pending := range status.Pending {
|
||||
plan.pending[pending.ID] = true
|
||||
}
|
||||
@@ -299,6 +306,14 @@ func (s *session) existingHost() *process {
|
||||
func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
|
||||
s := p.session
|
||||
live := false
|
||||
tasksReconciled := false
|
||||
catchUpReady := func() {
|
||||
if !tasksReconciled {
|
||||
s.reconcileHostTasks(plan.tasks)
|
||||
tasksReconciled = true
|
||||
}
|
||||
p.markReady()
|
||||
}
|
||||
goLive := func() {
|
||||
if live {
|
||||
return
|
||||
@@ -336,7 +351,7 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
|
||||
goLive()
|
||||
}
|
||||
if plan.catchUpEnd <= plan.base {
|
||||
p.markReady()
|
||||
catchUpReady()
|
||||
}
|
||||
sinceAck := 0
|
||||
for {
|
||||
@@ -344,9 +359,16 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
|
||||
if err != nil {
|
||||
goLive()
|
||||
p.markReady()
|
||||
if next := p.hostLost(conn, err); next != nil {
|
||||
if next, status := p.hostLost(conn, err); next != nil {
|
||||
conn = next
|
||||
plan = replayPlan{}
|
||||
plan = replayPlan{catchUpEnd: status.End, tasks: status.Tasks}
|
||||
tasksReconciled = false
|
||||
p.link.mu.Lock()
|
||||
committed := p.link.committed
|
||||
p.link.mu.Unlock()
|
||||
if committed >= status.End {
|
||||
catchUpReady()
|
||||
}
|
||||
continue
|
||||
}
|
||||
return
|
||||
@@ -371,7 +393,7 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
|
||||
goLive()
|
||||
}
|
||||
if frame.End >= plan.catchUpEnd {
|
||||
p.markReady()
|
||||
catchUpReady()
|
||||
}
|
||||
case claudehost.OpExit:
|
||||
goLive()
|
||||
@@ -416,19 +438,18 @@ func (p *process) markReady() {
|
||||
|
||||
// hostLost handles an ended host connection. A deliberate detach or close just
|
||||
// ends the reader; an unexpected loss is retried once (the host may only have
|
||||
// dropped the connection), and otherwise the host is treated as crashed: what
|
||||
// it left is stopped and the turn is settled with a notice, and the next prompt
|
||||
// resumes the session with --resume as after any release.
|
||||
func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn {
|
||||
// dropped the connection). A live host/CLI is preserved when retries fail;
|
||||
// only a confirmed dead reference is removed before allowing a resumed CLI.
|
||||
func (p *process) hostLost(conn *claudehost.Conn, cause error) (*claudehost.Conn, *claudehost.Status) {
|
||||
link := p.link
|
||||
_ = conn.Close()
|
||||
if link.detached.Load() {
|
||||
p.finishDetached()
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
if link.closing.Load() {
|
||||
p.finish(nil)
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
link.mu.Lock()
|
||||
committed := link.committed
|
||||
@@ -438,34 +459,54 @@ func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn
|
||||
if link.detached.Load() || link.closing.Load() {
|
||||
break
|
||||
}
|
||||
next, _, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout)
|
||||
next, status, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout)
|
||||
if err == nil {
|
||||
link.mu.Lock()
|
||||
link.conn = next
|
||||
link.mu.Unlock()
|
||||
return next
|
||||
return next, status
|
||||
}
|
||||
}
|
||||
if link.detached.Load() {
|
||||
p.finishDetached()
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
if link.closing.Load() {
|
||||
p.finish(nil)
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
log.Printf("claude chat %s: host connection lost (%v); falling back to --resume", p.session.id, cause)
|
||||
claudehost.KillStale(link.dir, link.record)
|
||||
if hostReferenceLive(link.record) {
|
||||
message := "Claude Code control connection is temporarily unavailable; work was left running, retry to reconnect"
|
||||
log.Printf("claude chat %s: host connection lost (%v); retaining live host and CLI", p.session.id, cause)
|
||||
s := p.session
|
||||
s.mu.Lock()
|
||||
if s.proc == p {
|
||||
s.proc = nil
|
||||
s.lastErr = message
|
||||
}
|
||||
s.mu.Unlock()
|
||||
p.finishDetachedReason(message)
|
||||
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventTool,
|
||||
Part: &protocol.ChatPart{Type: protocol.ChatPartTool, ID: "claude-host-unavailable", Name: "Claude Code",
|
||||
State: &protocol.ChatToolState{Status: protocol.ChatToolError, Error: &protocol.ChatError{Type: "connection", Message: message}}}})
|
||||
return nil, nil
|
||||
}
|
||||
log.Printf("claude chat %s: host and CLI stopped (%v); the next prompt resumes", p.session.id, cause)
|
||||
claudehost.Remove(link.dir, link.record.SessionID)
|
||||
p.mu.Lock()
|
||||
p.stderr.Reset()
|
||||
p.mu.Unlock()
|
||||
p.finish(errors.New("the Claude Code host stopped unexpectedly; the next prompt resumes this session with --resume"))
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// finishDetached ends a process this server let go of on shutdown. The CLI
|
||||
// keeps running in its host; nothing is reported to watchers.
|
||||
func (p *process) finishDetached() {
|
||||
p.finishDetachedReason("the server is shutting down")
|
||||
}
|
||||
|
||||
func (p *process) finishDetachedReason(reason string) {
|
||||
p.mu.Lock()
|
||||
if p.exited {
|
||||
p.mu.Unlock()
|
||||
@@ -478,7 +519,7 @@ func (p *process) finishDetached() {
|
||||
p.mu.Unlock()
|
||||
for _, ch := range pending {
|
||||
select {
|
||||
case ch <- controlReply{err: "the server is shutting down"}:
|
||||
case ch <- controlReply{err: reason}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
)
|
||||
|
||||
// hostReferenceLive fails closed: an alive or uninspectable PID must never be
|
||||
// signalled merely because its control socket did not answer. A stale reference
|
||||
// to a recycled PID can delay recovery, but cannot kill unrelated work.
|
||||
func hostReferenceLive(record claudehost.Record) bool {
|
||||
for _, pid := range []int{record.HostPID, record.ClaudePID} {
|
||||
if pid <= 1 {
|
||||
continue
|
||||
}
|
||||
err := syscall.Kill(pid, 0)
|
||||
if err != nil && !errors.Is(err, syscall.EPERM) {
|
||||
continue
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
|
||||
state, inspectErr := exec.CommandContext(ctx, "ps", "-o", "stat=", "-p", fmt.Sprint(pid)).Output()
|
||||
cancel()
|
||||
if inspectErr == nil && strings.HasPrefix(strings.TrimSpace(string(state)), "Z") {
|
||||
continue // a zombie cannot own live pipes or execute work
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Chat) removeDeadHostReference(record claudehost.Record) {
|
||||
if !hostReferenceLive(record) {
|
||||
claudehost.Remove(c.hostDir, record.SessionID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// reconcileHostTasks runs only after replay reaches the status snapshot's End.
|
||||
// It restores native identities even when their starting records were trimmed
|
||||
// from the bounded spool. Old hosts omit Tasks, which means unknown, not idle.
|
||||
func (s *session) reconcileHostTasks(tasks *[]claudehost.Task) {
|
||||
if tasks == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
present := make(map[string]bool, len(*tasks))
|
||||
var frames []protocol.ChatShell
|
||||
for _, task := range *tasks {
|
||||
id := task.ToolUseID
|
||||
if id == "" && task.TaskID != "" {
|
||||
id = "task:" + task.TaskID
|
||||
}
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
present[id] = true
|
||||
command := boundedText(task.Command, 64<<10)
|
||||
if command == "" && (task.Name == "Agent" || task.Name == "Task") && task.Description != "" {
|
||||
command = "Agent: " + boundedText(task.Description, 2048)
|
||||
}
|
||||
s.recordShellLocked(id, command)
|
||||
snapshot := s.shells[id]
|
||||
snapshot.background = true
|
||||
snapshot.taskID = task.TaskID
|
||||
snapshot.shell.Status = protocol.ChatShellRunning
|
||||
snapshot.shell.Exit = nil
|
||||
if task.Description != "" {
|
||||
snapshot.shell.Description = boundedText(task.Description, 2048)
|
||||
}
|
||||
if task.TaskType != "" {
|
||||
snapshot.shell.TaskType = boundedText(task.TaskType, 128)
|
||||
} else if snapshot.shell.TaskType == "" {
|
||||
switch task.Name {
|
||||
case "Bash":
|
||||
snapshot.shell.TaskType = "shell"
|
||||
case "Agent", "Task":
|
||||
snapshot.shell.TaskType = "agent"
|
||||
}
|
||||
}
|
||||
if snapshot.shell.StartedAt == 0 {
|
||||
if stamp, ok := transcriptTime(task.Timestamp); ok && stamp > 0 {
|
||||
snapshot.shell.StartedAt = stamp
|
||||
}
|
||||
}
|
||||
frames = append(frames, snapshot.shell)
|
||||
}
|
||||
for id, snapshot := range s.shells {
|
||||
if !snapshot.background || snapshot.shell.Status != protocol.ChatShellRunning || present[id] {
|
||||
continue
|
||||
}
|
||||
// The host says this background identity is no longer running, but
|
||||
// supplies neither an exit code nor a completion time. Keep both absent.
|
||||
snapshot.shell.Status = protocol.ChatShellExited
|
||||
frames = append(frames, snapshot.shell)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for i := range frames {
|
||||
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventShell, Shell: &frames[i]})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestHostTaskSnapshotDistinguishesLegacyUnknownFromKnownIdle(t *testing.T) {
|
||||
chat, s := backgroundSession(t)
|
||||
t.Cleanup(chat.Close)
|
||||
s.handleLine(nil, toolUse(t, "native-tool", "Bash", map[string]any{"command": "sleep 600", "run_in_background": true}))
|
||||
s.reconcileHostTasks(nil)
|
||||
if shells := running(t, s.chat); len(shells) != 1 {
|
||||
t.Fatalf("legacy unknown must preserve task liveness: %+v", shells)
|
||||
}
|
||||
empty := []claudehost.Task{}
|
||||
s.reconcileHostTasks(&empty)
|
||||
if shells := running(t, s.chat); len(shells) != 0 {
|
||||
t.Fatalf("authoritative idle must settle replayed background state: %+v", shells)
|
||||
}
|
||||
for _, source := range []string{`{}`, `{"tasks":[]}`} {
|
||||
var status claudehost.Status
|
||||
if err := json.Unmarshal([]byte(source), &status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if (status.Tasks == nil) != (source == `{}`) {
|
||||
t.Fatalf("capability presence was lost: %s %+v", source, status.Tasks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReattachRestoresTasksAfterTheirStartWasTrimmed(t *testing.T) {
|
||||
base, err := os.MkdirTemp("/tmp", "pct")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(base)
|
||||
socket := filepath.Join(base, "host.sock")
|
||||
listener, err := net.Listen("unix", socket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
current := []claudehost.Task{{ToolUseID: "native-bg", TaskID: "task-17", Name: "Bash", Command: "sleep 600", Description: "Wait for release", TaskType: "shell", Timestamp: "2026-10-22T12:00:00Z"}}
|
||||
status := claudehost.Status{SessionID: "trimmed-session", Base: 2000, Acked: 2000, End: 2300, Init: json.RawMessage(`{"models":[{"value":"fixture-model"}]}`), Tasks: ¤t}
|
||||
notify := make(chan struct{})
|
||||
stop := make(chan struct{})
|
||||
var stopOnce sync.Once
|
||||
stopServer := func() { stopOnce.Do(func() { close(stop) }) }
|
||||
defer stopServer()
|
||||
serverDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(serverDone)
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
var attach claudehost.Frame
|
||||
if json.NewDecoder(conn).Decode(&attach) != nil {
|
||||
return
|
||||
}
|
||||
out := json.NewEncoder(conn)
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpStatus, Status: &status})
|
||||
// The actual running task's initiating line is older than Base. The
|
||||
// retained spool contains a different, already-finished async tool.
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2000, End: 2100, Data: json.RawMessage(`{"type":"assistant","message":{"id":"historic","content":[{"type":"tool_use","id":"finished-tool","name":"Bash","input":{"command":"old job","run_in_background":true}}]}}`)})
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2100, End: 2300, Data: json.RawMessage(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"finished-tool","content":"Command running in background with ID: old"}]}}`)})
|
||||
select {
|
||||
case <-notify:
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","tool_use_id":"native-bg","task_id":"task-17","status":"completed","summary":"Finished"}`)})
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
<-stop
|
||||
}()
|
||||
chat := newChat(context.Background())
|
||||
defer chat.Close()
|
||||
p, conn, plan := chat.connectHost(claudehost.Record{SessionID: status.SessionID, Socket: socket})
|
||||
if p == nil {
|
||||
t.Fatal("fixture host did not attach")
|
||||
}
|
||||
s := p.session
|
||||
s.mu.Lock()
|
||||
s.proc = p
|
||||
s.mu.Unlock()
|
||||
go p.readHost(conn, plan)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
shells, err := chat.Shells(ctx, status.SessionID)
|
||||
if err != nil || len(shells) != 1 {
|
||||
t.Fatalf("restore authoritative native set after replay: %+v %v", shells, err)
|
||||
}
|
||||
want := protocol.ChatShell{ID: shellID("native-bg"), Status: protocol.ChatShellRunning, Command: "sleep 600", Description: "Wait for release", TaskType: "shell", StartedAt: time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC).UnixMilli()}
|
||||
if !reflect.DeepEqual(shells[0], want) {
|
||||
t.Fatalf("native identity and metadata changed:\n got %+v\nwant %+v", shells[0], want)
|
||||
}
|
||||
close(notify)
|
||||
eventually(t, "the restored task to finish under its original native id", func() bool {
|
||||
shells, _ := chat.Shells(ctx, status.SessionID)
|
||||
return len(shells) == 0
|
||||
})
|
||||
p.detach()
|
||||
stopServer()
|
||||
<-serverDone
|
||||
}
|
||||
|
||||
func TestLiveUnresponsiveHostSurvivesAttachFailureWithoutDuplicateCLI(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
record := onlyRecord(t, fixture)
|
||||
first.Detach()
|
||||
if err := syscall.Kill(record.HostPID, syscall.SIGSTOP); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer syscall.Kill(record.HostPID, syscall.SIGCONT)
|
||||
after := newChat(context.Background())
|
||||
defer after.Close()
|
||||
if p, _, _ := after.connectHost(record); p != nil {
|
||||
t.Fatal("a stopped fixture host unexpectedly answered")
|
||||
}
|
||||
if !alive(record.HostPID) || !alive(record.ClaudePID) {
|
||||
t.Fatal("control timeout killed live work")
|
||||
}
|
||||
if _, err := os.Stat(claudehost.PathsFor(fixture.hostDir, record.SessionID).Record); err != nil {
|
||||
t.Fatalf("live registration removed: %v", err)
|
||||
}
|
||||
s := after.sessionFor(session.ID, session.Directory)
|
||||
if _, err := s.startHosted(context.Background(), []string{"--session-id", session.ID}, "", nil); err == nil {
|
||||
t.Fatal("live unavailable session accepted a duplicate CLI start")
|
||||
}
|
||||
if err := syscall.Kill(record.HostPID, syscall.SIGCONT); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !after.attachHost(record) {
|
||||
t.Fatal("same host could not reconnect after it became responsive")
|
||||
}
|
||||
shells, err := after.Shells(context.Background(), session.ID)
|
||||
if err != nil || len(shells) != 1 || shells[0].ID != shellID("toolu_bg_fake") {
|
||||
t.Fatalf("reattachment changed running task: %+v %v", shells, err)
|
||||
}
|
||||
if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 {
|
||||
t.Fatalf("a control failure spawned a replacement CLI: %d", len(spawns))
|
||||
}
|
||||
}
|
||||
+43
-23
@@ -22,7 +22,7 @@ import (
|
||||
// Defaults for a host started without explicit limits.
|
||||
const (
|
||||
DefaultIdle = 20 * time.Minute
|
||||
DefaultMaxDetached = 24 * time.Hour
|
||||
DefaultMaxDetached = time.Duration(0)
|
||||
DefaultSpoolLimit = 64 << 20
|
||||
maxLineBytes = 32 << 20
|
||||
stderrRetained = 8 << 10
|
||||
@@ -85,24 +85,26 @@ type host struct {
|
||||
trans []transition
|
||||
busy bool
|
||||
// pending maps unanswered CLI control request ids to their offsets.
|
||||
pending map[string]int64
|
||||
initID string
|
||||
init json.RawMessage
|
||||
modelReqs map[string]string
|
||||
modeReqs map[string]string
|
||||
model string
|
||||
mode string
|
||||
exited bool
|
||||
exitErr string
|
||||
exitedAt time.Time
|
||||
stderr strings.Builder
|
||||
changed chan struct{}
|
||||
active *client
|
||||
detached time.Time
|
||||
activity time.Time
|
||||
stopping bool
|
||||
stopped chan struct{}
|
||||
claudeEnd chan struct{}
|
||||
tasks map[string]string // running native background tools, independent of foreground busy
|
||||
taskOrigins map[string]Task // native identity/metadata retained independently of spool compaction
|
||||
pending map[string]int64
|
||||
initID string
|
||||
init json.RawMessage
|
||||
modelReqs map[string]string
|
||||
modeReqs map[string]string
|
||||
model string
|
||||
mode string
|
||||
exited bool
|
||||
exitErr string
|
||||
exitedAt time.Time
|
||||
stderr strings.Builder
|
||||
changed chan struct{}
|
||||
active *client
|
||||
detached time.Time
|
||||
activity time.Time
|
||||
stopping bool
|
||||
stopped chan struct{}
|
||||
claudeEnd chan struct{}
|
||||
}
|
||||
|
||||
// Main runs a host. args are the host flags, then "--", then the Claude Code
|
||||
@@ -113,7 +115,7 @@ func Main(args []string) int {
|
||||
dir := flags.String("dir", "", "private host directory (registry, socket, spool)")
|
||||
id := flags.String("id", "", "Claude session id")
|
||||
idle := flags.Duration("idle", DefaultIdle, "release an idle CLI after this long with no server attached")
|
||||
maxDetached := flags.Duration("max-detached", DefaultMaxDetached, "release the CLI after this long with no server attached, even when busy")
|
||||
maxDetached := flags.Duration("max-detached", DefaultMaxDetached, "optional detached lifetime limit (0 keeps live sessions until explicitly closed)")
|
||||
spoolLimit := flags.Int64("spool-limit", DefaultSpoolLimit, "bytes of acknowledged output kept in the spool")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
@@ -341,6 +343,7 @@ func (h *host) appendLine(line []byte) {
|
||||
h.end += int64(len(data))
|
||||
h.lines = append(h.lines, lineRef{start: start, end: h.end})
|
||||
h.activity = time.Now()
|
||||
h.trackTasksLocked(line)
|
||||
switch env.Type {
|
||||
case "control_request":
|
||||
if env.RequestID != "" {
|
||||
@@ -425,10 +428,27 @@ func (h *host) statusLocked() *Status {
|
||||
}
|
||||
sort.Slice(pending, func(i, j int) bool { return pending[i].Offset < pending[j].Offset })
|
||||
cwd, _ := os.Getwd()
|
||||
tasks := make([]Task, 0, len(h.tasks))
|
||||
for key, taskID := range h.tasks {
|
||||
task := h.taskOrigins[key]
|
||||
if task.ToolUseID == "" && !strings.HasPrefix(key, "task:") {
|
||||
task.ToolUseID = key
|
||||
}
|
||||
if task.TaskID == "" {
|
||||
task.TaskID = taskID
|
||||
}
|
||||
tasks = append(tasks, task)
|
||||
}
|
||||
sort.Slice(tasks, func(i, j int) bool {
|
||||
if tasks[i].ToolUseID == tasks[j].ToolUseID {
|
||||
return tasks[i].TaskID < tasks[j].TaskID
|
||||
}
|
||||
return tasks[i].ToolUseID < tasks[j].ToolUseID
|
||||
})
|
||||
status := &Status{
|
||||
Version: Version, SessionID: h.id, HostPID: os.Getpid(),
|
||||
Base: h.base, End: h.end, Acked: h.acked, Busy: h.busy, BusyAtAcked: h.busyAtLocked(h.acked),
|
||||
Init: h.init, Model: h.model, Mode: h.mode, Pending: pending,
|
||||
Init: h.init, Model: h.model, Mode: h.mode, Pending: pending, Tasks: &tasks,
|
||||
Exited: h.exited, ExitErr: h.exitErr, Stderr: h.stderr.String(), Directory: cwd,
|
||||
}
|
||||
if h.cmd != nil && h.cmd.Process != nil {
|
||||
@@ -688,8 +708,8 @@ func (h *host) watchIdle() {
|
||||
now := time.Now()
|
||||
h.mu.Lock()
|
||||
attached := h.active != nil
|
||||
idle := !h.busy && len(h.pending) == 0 && now.Sub(h.detached) > h.idle && now.Sub(h.activity) > h.idle
|
||||
tooLong := now.Sub(h.detached) > h.maxDetached
|
||||
idle := !h.busy && len(h.pending) == 0 && len(h.tasks) == 0 && now.Sub(h.detached) > h.idle && now.Sub(h.activity) > h.idle
|
||||
tooLong := h.maxDetached > 0 && now.Sub(h.detached) > h.maxDetached
|
||||
exitedLong := h.exited && now.Sub(h.exitedAt) > h.idle
|
||||
h.mu.Unlock()
|
||||
if attached {
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
package claudehost
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// The host owns the real CLI even with no Pipelines connection. Foreground
|
||||
// result messages do not end background tools; only their native completion
|
||||
// notifications do. Keep this lifecycle state independently of spool acks.
|
||||
func (h *host) trackTasksLocked(line []byte) {
|
||||
var event struct {
|
||||
Type string `json:"type"`
|
||||
Subtype string `json:"subtype"`
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
TaskID string `json:"task_id"`
|
||||
Status string `json:"status"`
|
||||
Description string `json:"description"`
|
||||
TaskType string `json:"task_type"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Message struct {
|
||||
Content json.RawMessage `json:"content"`
|
||||
} `json:"message"`
|
||||
}
|
||||
if json.Unmarshal(line, &event) != nil {
|
||||
return
|
||||
}
|
||||
if h.tasks == nil {
|
||||
h.tasks = map[string]string{}
|
||||
}
|
||||
if h.taskOrigins == nil {
|
||||
h.taskOrigins = map[string]Task{}
|
||||
}
|
||||
switch event.Subtype {
|
||||
case "task_started", "task_progress":
|
||||
id := event.ToolUseID
|
||||
if id == "" {
|
||||
for key, task := range h.tasks {
|
||||
if event.TaskID != "" && task == event.TaskID {
|
||||
id = key
|
||||
break
|
||||
}
|
||||
}
|
||||
if id == "" {
|
||||
id = "task:" + event.TaskID
|
||||
}
|
||||
}
|
||||
if id != "task:" {
|
||||
if event.ToolUseID != "" && event.TaskID != "" {
|
||||
fallback := "task:" + event.TaskID
|
||||
if prior, exists := h.taskOrigins[fallback]; exists {
|
||||
if _, recorded := h.taskOrigins[id]; !recorded {
|
||||
h.taskOrigins[id] = prior
|
||||
}
|
||||
delete(h.tasks, fallback)
|
||||
delete(h.taskOrigins, fallback)
|
||||
}
|
||||
}
|
||||
h.tasks[id] = event.TaskID
|
||||
origin := h.taskOrigins[id]
|
||||
if event.ToolUseID != "" {
|
||||
origin.ToolUseID = event.ToolUseID
|
||||
}
|
||||
if event.TaskID != "" {
|
||||
origin.TaskID = event.TaskID
|
||||
}
|
||||
if event.Description != "" {
|
||||
origin.Description = taskText(event.Description, 2048)
|
||||
}
|
||||
if event.TaskType != "" {
|
||||
origin.TaskType = taskText(event.TaskType, 128)
|
||||
}
|
||||
if origin.Timestamp == "" {
|
||||
origin.Timestamp = event.Timestamp
|
||||
}
|
||||
h.taskOrigins[id] = origin
|
||||
}
|
||||
case "task_notification":
|
||||
if event.Status == "completed" || event.Status == "failed" || event.Status == "stopped" {
|
||||
for key, task := range h.tasks {
|
||||
if key == event.ToolUseID || (event.TaskID != "" && (task == event.TaskID || key == "task:"+event.TaskID)) {
|
||||
delete(h.tasks, key)
|
||||
delete(h.taskOrigins, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if event.Type == "result" {
|
||||
for key := range h.taskOrigins {
|
||||
if _, running := h.tasks[key]; !running {
|
||||
delete(h.taskOrigins, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
var blocks []taskBlock
|
||||
if json.Unmarshal(event.Message.Content, &blocks) != nil {
|
||||
var text string
|
||||
if json.Unmarshal(event.Message.Content, &text) != nil {
|
||||
return
|
||||
}
|
||||
blocks = []taskBlock{{Type: "text", Text: text}}
|
||||
}
|
||||
|
||||
finishText := func(text string) {
|
||||
for _, match := range completedTaskNotice.FindAllStringSubmatch(text, -1) {
|
||||
key := match[1]
|
||||
taskID := h.tasks[key]
|
||||
delete(h.tasks, key)
|
||||
delete(h.taskOrigins, key)
|
||||
if taskID != "" {
|
||||
delete(h.tasks, "task:"+taskID)
|
||||
delete(h.taskOrigins, "task:"+taskID)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, block := range blocks {
|
||||
switch block.Type {
|
||||
case "tool_use":
|
||||
if block.ID != "" && (block.Name == "Agent" || block.Name == "Task" || block.Name == "Bash") {
|
||||
if block.Input.Background {
|
||||
if _, exists := h.tasks[block.ID]; !exists {
|
||||
h.tasks[block.ID] = ""
|
||||
}
|
||||
}
|
||||
origin := h.taskOrigins[block.ID]
|
||||
origin.ToolUseID, origin.Name = block.ID, block.Name
|
||||
origin.Command = taskText(block.Input.Command, 64<<10)
|
||||
if block.Input.Description != "" {
|
||||
origin.Description = taskText(block.Input.Description, 2048)
|
||||
}
|
||||
if origin.TaskType == "" {
|
||||
if block.Name == "Bash" {
|
||||
origin.TaskType = "shell"
|
||||
} else {
|
||||
origin.TaskType = "agent"
|
||||
}
|
||||
}
|
||||
if origin.Timestamp == "" {
|
||||
origin.Timestamp = event.Timestamp
|
||||
}
|
||||
h.taskOrigins[block.ID] = origin
|
||||
}
|
||||
case "tool_result":
|
||||
var text string
|
||||
if json.Unmarshal(block.Content, &text) != nil {
|
||||
var parts []struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
_ = json.Unmarshal(block.Content, &parts)
|
||||
for _, part := range parts {
|
||||
text += part.Text + "\n"
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, "running in background with ID") || strings.HasPrefix(strings.TrimSpace(text), "Async agent launched") {
|
||||
if block.ToolUseID != "" {
|
||||
if _, exists := h.tasks[block.ToolUseID]; !exists {
|
||||
h.tasks[block.ToolUseID] = ""
|
||||
}
|
||||
}
|
||||
} else {
|
||||
delete(h.tasks, block.ToolUseID)
|
||||
delete(h.taskOrigins, block.ToolUseID)
|
||||
}
|
||||
finishText(text)
|
||||
case "text":
|
||||
finishText(block.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var completedTaskNotice = regexp.MustCompile(`(?s)<task-notification>.*?<tool-use-id>\s*([^<\s]+)\s*</tool-use-id>.*?</task-notification>`)
|
||||
|
||||
// Native stream-json message content can be an array or a plain notification.
|
||||
type taskBlock struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
Name string `json:"name"`
|
||||
Input struct {
|
||||
Background bool `json:"run_in_background"`
|
||||
Command string `json:"command"`
|
||||
Description string `json:"description"`
|
||||
} `json:"input"`
|
||||
Text string `json:"text"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
}
|
||||
|
||||
func taskText(s string, limit int) string {
|
||||
if len(s) <= limit {
|
||||
return s
|
||||
}
|
||||
for limit > 0 && !utf8.ValidString(s[:limit]) {
|
||||
limit--
|
||||
}
|
||||
return s[:limit]
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package claudehost
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNativeTaskSnapshotOutlivesForegroundAndSpool(t *testing.T) {
|
||||
h := &host{}
|
||||
h.trackTasksLocked([]byte(`{"type":"assistant","timestamp":"2026-11-07T12:00:00Z","message":{"content":[{"type":"tool_use","id":"tool-bg","name":"Bash","input":{"command":"sleep 600","run_in_background":true}}]}}`))
|
||||
h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_started","tool_use_id":"tool-bg","task_id":"native-bg","description":"Background build","task_type":"local_bash"}`))
|
||||
h.trackTasksLocked([]byte(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool-bg","content":"Command running in background with ID: native-bg"}]}}`))
|
||||
h.trackTasksLocked([]byte(`{"type":"result"}`))
|
||||
h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_progress","task_id":"native-bg"}`))
|
||||
// Acknowledged stream data may be compacted; live membership must not be.
|
||||
h.base, h.end, h.acked = 10000, 10000, 10000
|
||||
h.lines = nil
|
||||
status := h.statusLocked()
|
||||
if status.Tasks == nil || len(*status.Tasks) != 1 {
|
||||
t.Fatalf("native snapshot: %+v", status.Tasks)
|
||||
}
|
||||
task := (*status.Tasks)[0]
|
||||
if task.ToolUseID != "tool-bg" || task.TaskID != "native-bg" || task.Command != "sleep 600" || task.Description != "Background build" || task.Timestamp != "2026-11-07T12:00:00Z" {
|
||||
t.Fatalf("identity/metadata lost: %+v", task)
|
||||
}
|
||||
h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_notification","task_id":"native-bg","status":"completed"}`))
|
||||
if status = h.statusLocked(); status.Tasks == nil || len(*status.Tasks) != 0 {
|
||||
t.Fatalf("completion not authoritative: %+v", status.Tasks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeTaskAliasesAndTextCompletion(t *testing.T) {
|
||||
h := &host{}
|
||||
h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_progress","task_id":"native-agent"}`))
|
||||
h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_started","task_id":"native-agent","tool_use_id":"tool-agent","description":"Review"}`))
|
||||
h.trackTasksLocked([]byte(`{"type":"assistant","message":{"content":[{"type":"tool_use","id":"tool-agent","name":"Agent","input":{"description":"Review"}}]}}`))
|
||||
h.trackTasksLocked([]byte(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool-agent","content":[{"type":"text","text":"Async agent launched successfully."}]}]}}`))
|
||||
if len(h.tasks) != 1 || h.tasks["tool-agent"] != "native-agent" {
|
||||
t.Fatalf("alias lost or duplicated: %+v", h.tasks)
|
||||
}
|
||||
h.trackTasksLocked([]byte(`{"type":"user","message":{"content":"<task-notification><tool-use-id>tool-agent</tool-use-id><status>completed</status></task-notification>"}}`))
|
||||
if len(h.tasks) != 0 || len(h.taskOrigins) != 0 {
|
||||
t.Fatalf("text completion retained native work: %+v %+v", h.tasks, h.taskOrigins)
|
||||
}
|
||||
h.trackTasksLocked([]byte(`{"type":"assistant","message":{"content":[{"type":"tool_use","id":"foreground","name":"Agent","input":{"description":"Sync review"}}]}}`))
|
||||
h.trackTasksLocked([]byte(`{"type":"result"}`))
|
||||
if len(h.tasks) != 0 || len(h.taskOrigins) != 0 {
|
||||
t.Fatal("foreground task leaked into detached liveness")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetachedBackgroundCLIIsNotReapedAsIdle(t *testing.T) {
|
||||
dir, err := os.MkdirTemp("/tmp", "phbg")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
now := time.Now()
|
||||
h := &host{dir: dir, id: "background-survival", paths: PathsFor(dir, "background-survival"), idle: 50 * time.Millisecond, maxDetached: DefaultMaxDetached, spoolLimit: 64 << 20, pending: map[string]int64{}, modelReqs: map[string]string{}, modeReqs: map[string]string{}, changed: make(chan struct{}), stopped: make(chan struct{}), claudeEnd: make(chan struct{}), activity: now, detached: now}
|
||||
// This is an actual CLI process owned by a separate host lifecycle, not a
|
||||
// synthetic task row. It remains quiet after finishing its foreground turn.
|
||||
script := `printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"tool_use","id":"tool-bg","name":"Bash","input":{"command":"quiet work","run_in_background":true}}]}}' '{"type":"result"}'; sleep 30`
|
||||
if err := h.start([]string{"/bin/sh", "-c", script}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer h.shutdown("test cleanup")
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
h.mu.Lock()
|
||||
ready := len(h.tasks) == 1 && !h.busy
|
||||
h.mu.Unlock()
|
||||
if ready {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
go h.watchIdle()
|
||||
time.Sleep(180 * time.Millisecond)
|
||||
select {
|
||||
case <-h.stopped:
|
||||
t.Fatal("detached background-only CLI was killed by idle expiry")
|
||||
default:
|
||||
}
|
||||
if syscall.Kill(h.cmd.Process.Pid, 0) != nil {
|
||||
t.Fatal("native CLI did not survive detached idle window")
|
||||
}
|
||||
t.Logf("native CLI PID=%d survived detached background-only idle window", h.cmd.Process.Pid)
|
||||
h.mu.Lock()
|
||||
h.trackTasksLocked([]byte(`{"type":"system","subtype":"task_notification","tool_use_id":"tool-bg","status":"completed"}`))
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case <-h.stopped:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("completed idle CLI was not released")
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,19 @@ type Pending struct {
|
||||
Offset int64 `json:"offset"`
|
||||
}
|
||||
|
||||
// Task is a native background tool still owned by the hosted CLI. It is kept
|
||||
// separately from the bounded output spool, so trimming old output cannot erase
|
||||
// the identity needed by a reattaching server. Missing metadata stays absent.
|
||||
type Task struct {
|
||||
ToolUseID string `json:"tool_use_id,omitempty"`
|
||||
TaskID string `json:"task_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
TaskType string `json:"task_type,omitempty"`
|
||||
Command string `json:"command,omitempty"`
|
||||
Timestamp string `json:"timestamp,omitempty"`
|
||||
}
|
||||
|
||||
// Status is the host's snapshot of its CLI.
|
||||
type Status struct {
|
||||
Version int `json:"version"`
|
||||
@@ -84,6 +97,10 @@ type Status struct {
|
||||
// live processing resumes.
|
||||
Busy bool `json:"busy"`
|
||||
BusyAtAcked bool `json:"busy_at_acked"`
|
||||
// Tasks is the authoritative running background set at End. Nil means an
|
||||
// older host did not report this capability; a nonnil empty list means it
|
||||
// knows that no background tasks remain. Reconcile after replay reaches End.
|
||||
Tasks *[]Task `json:"tasks,omitempty"`
|
||||
// Init is the CLI's initialize response (models, account), so a
|
||||
// reconnecting server does not initialize the CLI twice.
|
||||
Init json.RawMessage `json:"init,omitempty"`
|
||||
|
||||
Reference in New Issue
Block a user