From 723ed3aa6842e2a5b00186d90840c41c6874f520 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 7 Nov 2026 16:28:26 +0000 Subject: [PATCH] Prove task recovery beyond spool trim and order live events after snapshots --- internal/claude/hosted.go | 27 ++++- internal/claude/hosted_tasks.go | 41 ++++++++ internal/claude/hosted_tasks_boundary_test.go | 99 +++++++++++++++++++ internal/claude/hosted_tasks_test.go | 89 ++++++++++++++++- 4 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 internal/claude/hosted_tasks_boundary_test.go diff --git a/internal/claude/hosted.go b/internal/claude/hosted.go index 2487d693..d7a83b55 100644 --- a/internal/claude/hosted.go +++ b/internal/claude/hosted.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "log" + "net" "os" "os/exec" "path/filepath" @@ -180,7 +181,24 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e // is never killed to repair the control connection. Its registration blocks a // duplicate CLI until reconnect succeeds. Only dead references are removed. func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) { - conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout) + var conn *claudehost.Conn + var status *claudehost.Status + var err error + for attempt := 0; attempt < 3; attempt++ { + conn, status, err = claudehost.Attach(record.Socket, 0, hostAttachTimeout) + if err == nil { + break + } + var networkError net.Error + if !hostReferenceLive(record) || (errors.As(err, &networkError) && networkError.Timeout()) { + break + } + // A pre-timeout attach can still be draining when a paused host + // resumes. Retry transient EOF/refusal without replacing its CLI. + if attempt < 2 { + time.Sleep(100 * time.Millisecond) + } + } if err != nil { log.Printf("claude chat %s: host unreachable (%v); retaining any live work for reconnect", record.SessionID, err) c.removeDeadHostReference(record) @@ -375,11 +393,16 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { } switch frame.Op { case claudehost.OpOut: + // Delivery can skip old records after spool compaction. A frame + // starting at/after End is live: reconcile the older snapshot first. + if frame.Offset >= plan.catchUpEnd { + catchUpReady() + } if frame.Offset >= plan.silentUntil { goLive() } if !(frame.Offset < plan.catchUpEnd && answeredRequest(frame.Data, plan.pending)) { - s.handleLine(p, frame.Data) + s.handleLine(p, s.resolveHostedTaskLine(frame.Data)) } p.link.mu.Lock() p.link.committed = frame.End diff --git a/internal/claude/hosted_tasks.go b/internal/claude/hosted_tasks.go index 8caf47a1..0100b618 100644 --- a/internal/claude/hosted_tasks.go +++ b/internal/claude/hosted_tasks.go @@ -1,6 +1,8 @@ package claude import ( + "bytes" + "encoding/json" "git.a-wan.net/alan/pipelines/internal/claudehost" "git.a-wan.net/alan/pipelines/internal/protocol" ) @@ -68,3 +70,42 @@ func (s *session) reconcileHostTasks(tasks *[]claudehost.Task) { s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventShell, Shell: &frames[i]}) } } + +// resolveHostedTaskLine correlates a completion that carries only the native +// task id with the tool identity restored from the authoritative snapshot. +// This is translation inside the server; no altered event is sent to the CLI. +func (s *session) resolveHostedTaskLine(line []byte) []byte { + if !bytes.Contains(line, []byte(`"task_notification"`)) { + return line + } + var event struct { + Subtype string `json:"subtype"` + TaskID string `json:"task_id"` + ToolUseID string `json:"tool_use_id"` + } + if json.Unmarshal(line, &event) != nil || event.Subtype != "task_notification" || event.TaskID == "" || event.ToolUseID != "" { + return line + } + s.mu.Lock() + id := "" + for toolID, snapshot := range s.shells { + if snapshot.taskID == event.TaskID { + id = toolID + break + } + } + s.mu.Unlock() + if id == "" { + return line + } + var fields map[string]json.RawMessage + if json.Unmarshal(line, &fields) != nil { + return line + } + fields["tool_use_id"], _ = json.Marshal(id) + normalized, err := json.Marshal(fields) + if err != nil { + return line + } + return normalized +} diff --git a/internal/claude/hosted_tasks_boundary_test.go b/internal/claude/hosted_tasks_boundary_test.go new file mode 100644 index 00000000..ba4f4a80 --- /dev/null +++ b/internal/claude/hosted_tasks_boundary_test.go @@ -0,0 +1,99 @@ +package claude + +import ( + "context" + "encoding/json" + "net" + "os" + "path/filepath" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" +) + +func TestLiveTaskEventAfterTrimmedReplayBoundaryWinsOverSnapshot(t *testing.T) { + for _, tc := range []struct { + name string + tasks []claudehost.Task + line string + want int + }{ + { + name: "new task remains running after older idle snapshot", + tasks: []claudehost.Task{}, + line: `{"type":"assistant","message":{"id":"fresh","content":[{"type":"tool_use","id":"fresh-tool","name":"Bash","input":{"command":"new job","run_in_background":true}}]}}`, + want: 1, + }, + { + name: "completion is not resurrected by older running snapshot", + tasks: []claudehost.Task{{ToolUseID: "current-tool", TaskID: "native-task", Name: "Bash", Command: "existing job"}}, + line: `{"type":"system","subtype":"task_notification","task_id":"native-task","status":"completed","summary":"Finished"}`, + want: 0, + }, + } { + t.Run(tc.name, func(t *testing.T) { + base, err := os.MkdirTemp("/tmp", "pcb") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(base) + socket := filepath.Join(base, "host.sock") + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + var attach claudehost.Frame + if json.NewDecoder(conn).Decode(&attach) != nil { + return + } + out := json.NewEncoder(conn) + _ = out.Encode(claudehost.Frame{Op: claudehost.OpStatus, Status: &claudehost.Status{SessionID: "boundary-session", Base: 100, End: 200, Acked: 100, Init: json.RawMessage(`{"models":[{"value":"fixture"}]}`), Tasks: &tc.tasks}}) + // The old replay region was compacted before delivery. Its + // first available record is newer than the snapshot boundary. + _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 300, End: 400, Data: json.RawMessage(tc.line)}) + <-stop + }() + chat := newChat(context.Background()) + defer chat.Close() + p, conn, plan := chat.connectHost(claudehost.Record{SessionID: "boundary-session", Socket: socket}) + if p == nil { + close(stop) + t.Fatal("fixture did not attach") + } + s := p.session + s.mu.Lock() + s.proc = p + s.mu.Unlock() + go p.readHost(conn, plan) + defer func() { p.detach(); close(stop); <-done }() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, _ = chat.Shells(ctx, "boundary-session") + p.link.mu.Lock() + committed := p.link.committed + p.link.mu.Unlock() + // Shells can unblock just before the first live line completes. + eventually(t, "the first live record to be committed", func() bool { + p.link.mu.Lock() + committed = p.link.committed + p.link.mu.Unlock() + return committed == 400 + }) + shells, err := chat.Shells(ctx, "boundary-session") + if err != nil || len(shells) != tc.want { + t.Fatalf("live record lost to stale snapshot: %+v %v", shells, err) + } + }) + } +} diff --git a/internal/claude/hosted_tasks_test.go b/internal/claude/hosted_tasks_test.go index 27bbb453..92db5d6f 100644 --- a/internal/claude/hosted_tasks_test.go +++ b/internal/claude/hosted_tasks_test.go @@ -1,12 +1,14 @@ package claude import ( + "bytes" "context" "encoding/json" "net" "os" "path/filepath" "reflect" + "strings" "sync" "syscall" "testing" @@ -79,7 +81,7 @@ func TestReattachRestoresTasksAfterTheirStartWasTrimmed(t *testing.T) { _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2100, End: 2300, Data: json.RawMessage(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"finished-tool","content":"Command running in background with ID: old"}]}}`)}) select { case <-notify: - _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","tool_use_id":"native-bg","task_id":"task-17","status":"completed","summary":"Finished"}`)}) + _ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","task_id":"task-17","status":"completed","summary":"Finished"}`)}) case <-stop: return } @@ -160,3 +162,88 @@ func TestLiveUnresponsiveHostSurvivesAttachFailureWithoutDuplicateCLI(t *testing t.Fatalf("a control failure spawned a replacement CLI: %d", len(spawns)) } } + +func TestNativeHostRetainsTaskIdentityBeyondSpoolCompactionAndRestart(t *testing.T) { + fixture := hostedSetup(t) + // Exercise the real host compactor rather than retaining all initiating + // events. The fake CLI supplies actual stream metadata, not test-side state. + hostConfig.Lock() + oldArgs := append([]string(nil), hostConfig.args...) + hostConfig.args = []string{"-spool-limit", "512"} + hostConfig.Unlock() + t.Cleanup(func() { + hostConfig.Lock() + hostConfig.args = oldArgs + hostConfig.Unlock() + }) + script := filepath.Join(filepath.Dir(fixture.hostDir), "claude") + raw, err := os.ReadFile(script) + if err != nil { + t.Fatal(err) + } + raw = []byte(strings.Replace(string(raw), `"input": {"command": "sleep 600", "run_in_background": True}`, `"input": {"command": "sleep 600", "run_in_background": True, "description": "Release watcher"}`, 1)) + raw = []byte(strings.Replace(string(raw), `send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}})`, `send({"type": "assistant", "timestamp": stamp(), "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}})`, 1)) + if err := os.WriteFile(script, raw, 0o700); err != nil { + t.Fatal(err) + } + first := hostedChat(t) + session := newCreatedChat(t, first) + frames := subscribe(t, first, session.ID) + if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + before, err := first.Shells(context.Background(), session.ID) + if err != nil || len(before) != 1 || before[0].StartedAt == 0 || before[0].Description != "Release watcher" { + t.Fatalf("native task metadata was not observed: %+v %v", before, err) + } + record := onlyRecord(t, fixture) + trimmed := false + for i := 0; i < 10; i++ { + if err := first.Prompt(context.Background(), session.ID, "hello"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + spool, err := os.ReadFile(record.Spool) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(spool, []byte("toolu_bg_fake")) { + trimmed = true + break + } + } + if !trimmed { + t.Fatal("fixture did not compact the task's initiating records") + } + first.Detach() + control, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout) + if err != nil { + t.Fatal(err) + } + _ = control.Close() + if status.Base <= 0 || status.Tasks == nil || len(*status.Tasks) != 1 { + t.Fatalf("task did not survive in authoritative host state: %+v", status) + } + second := hostedChat(t) + after, err := second.Shells(context.Background(), session.ID) + if err != nil || !reflect.DeepEqual(after, before) { + t.Fatalf("spool compaction/restart changed task identity or metadata:\nbefore=%+v\nafter=%+v\nerr=%v", before, after, err) + } + reused := onlyRecord(t, fixture) + if reused.HostPID != record.HostPID || reused.ClaudePID != record.ClaudePID { + t.Fatal("reattachment replaced the live host or CLI") + } + if calls := readArgs(t, fixture.logPath); len(calls) != 1 { + t.Fatalf("reattachment spawned another CLI: %d", len(calls)) + } + frames = subscribe(t, second, session.ID) + if err := second.Prompt(context.Background(), session.ID, "notify me"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + finished, err := second.Shells(context.Background(), session.ID) + if err != nil || len(finished) != 0 { + t.Fatalf("native completion did not settle the restored task: %+v %v", finished, err) + } +}