Prove task recovery beyond spool trim and order live events after snapshots

This commit is contained in:
2026-11-07 16:28:26 +00:00
parent 5d38ba3352
commit 723ed3aa68
4 changed files with 253 additions and 3 deletions
+25 -2
View File
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"log"
"net"
"os"
"os/exec"
"path/filepath"
@@ -180,7 +181,24 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e
// is never killed to repair the control connection. Its registration blocks a
// duplicate CLI until reconnect succeeds. Only dead references are removed.
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
var conn *claudehost.Conn
var status *claudehost.Status
var err error
for attempt := 0; attempt < 3; attempt++ {
conn, status, err = claudehost.Attach(record.Socket, 0, hostAttachTimeout)
if err == nil {
break
}
var networkError net.Error
if !hostReferenceLive(record) || (errors.As(err, &networkError) && networkError.Timeout()) {
break
}
// A pre-timeout attach can still be draining when a paused host
// resumes. Retry transient EOF/refusal without replacing its CLI.
if attempt < 2 {
time.Sleep(100 * time.Millisecond)
}
}
if err != nil {
log.Printf("claude chat %s: host unreachable (%v); retaining any live work for reconnect", record.SessionID, err)
c.removeDeadHostReference(record)
@@ -375,11 +393,16 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
}
switch frame.Op {
case claudehost.OpOut:
// Delivery can skip old records after spool compaction. A frame
// starting at/after End is live: reconcile the older snapshot first.
if frame.Offset >= plan.catchUpEnd {
catchUpReady()
}
if frame.Offset >= plan.silentUntil {
goLive()
}
if !(frame.Offset < plan.catchUpEnd && answeredRequest(frame.Data, plan.pending)) {
s.handleLine(p, frame.Data)
s.handleLine(p, s.resolveHostedTaskLine(frame.Data))
}
p.link.mu.Lock()
p.link.committed = frame.End
+41
View File
@@ -1,6 +1,8 @@
package claude
import (
"bytes"
"encoding/json"
"git.a-wan.net/alan/pipelines/internal/claudehost"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
@@ -68,3 +70,42 @@ func (s *session) reconcileHostTasks(tasks *[]claudehost.Task) {
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventShell, Shell: &frames[i]})
}
}
// resolveHostedTaskLine correlates a completion that carries only the native
// task id with the tool identity restored from the authoritative snapshot.
// This is translation inside the server; no altered event is sent to the CLI.
func (s *session) resolveHostedTaskLine(line []byte) []byte {
if !bytes.Contains(line, []byte(`"task_notification"`)) {
return line
}
var event struct {
Subtype string `json:"subtype"`
TaskID string `json:"task_id"`
ToolUseID string `json:"tool_use_id"`
}
if json.Unmarshal(line, &event) != nil || event.Subtype != "task_notification" || event.TaskID == "" || event.ToolUseID != "" {
return line
}
s.mu.Lock()
id := ""
for toolID, snapshot := range s.shells {
if snapshot.taskID == event.TaskID {
id = toolID
break
}
}
s.mu.Unlock()
if id == "" {
return line
}
var fields map[string]json.RawMessage
if json.Unmarshal(line, &fields) != nil {
return line
}
fields["tool_use_id"], _ = json.Marshal(id)
normalized, err := json.Marshal(fields)
if err != nil {
return line
}
return normalized
}
@@ -0,0 +1,99 @@
package claude
import (
"context"
"encoding/json"
"net"
"os"
"path/filepath"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/claudehost"
)
func TestLiveTaskEventAfterTrimmedReplayBoundaryWinsOverSnapshot(t *testing.T) {
for _, tc := range []struct {
name string
tasks []claudehost.Task
line string
want int
}{
{
name: "new task remains running after older idle snapshot",
tasks: []claudehost.Task{},
line: `{"type":"assistant","message":{"id":"fresh","content":[{"type":"tool_use","id":"fresh-tool","name":"Bash","input":{"command":"new job","run_in_background":true}}]}}`,
want: 1,
},
{
name: "completion is not resurrected by older running snapshot",
tasks: []claudehost.Task{{ToolUseID: "current-tool", TaskID: "native-task", Name: "Bash", Command: "existing job"}},
line: `{"type":"system","subtype":"task_notification","task_id":"native-task","status":"completed","summary":"Finished"}`,
want: 0,
},
} {
t.Run(tc.name, func(t *testing.T) {
base, err := os.MkdirTemp("/tmp", "pcb")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(base)
socket := filepath.Join(base, "host.sock")
listener, err := net.Listen("unix", socket)
if err != nil {
t.Fatal(err)
}
defer listener.Close()
stop := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
conn, err := listener.Accept()
if err != nil {
return
}
defer conn.Close()
var attach claudehost.Frame
if json.NewDecoder(conn).Decode(&attach) != nil {
return
}
out := json.NewEncoder(conn)
_ = out.Encode(claudehost.Frame{Op: claudehost.OpStatus, Status: &claudehost.Status{SessionID: "boundary-session", Base: 100, End: 200, Acked: 100, Init: json.RawMessage(`{"models":[{"value":"fixture"}]}`), Tasks: &tc.tasks}})
// The old replay region was compacted before delivery. Its
// first available record is newer than the snapshot boundary.
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 300, End: 400, Data: json.RawMessage(tc.line)})
<-stop
}()
chat := newChat(context.Background())
defer chat.Close()
p, conn, plan := chat.connectHost(claudehost.Record{SessionID: "boundary-session", Socket: socket})
if p == nil {
close(stop)
t.Fatal("fixture did not attach")
}
s := p.session
s.mu.Lock()
s.proc = p
s.mu.Unlock()
go p.readHost(conn, plan)
defer func() { p.detach(); close(stop); <-done }()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
_, _ = chat.Shells(ctx, "boundary-session")
p.link.mu.Lock()
committed := p.link.committed
p.link.mu.Unlock()
// Shells can unblock just before the first live line completes.
eventually(t, "the first live record to be committed", func() bool {
p.link.mu.Lock()
committed = p.link.committed
p.link.mu.Unlock()
return committed == 400
})
shells, err := chat.Shells(ctx, "boundary-session")
if err != nil || len(shells) != tc.want {
t.Fatalf("live record lost to stale snapshot: %+v %v", shells, err)
}
})
}
}
+88 -1
View File
@@ -1,12 +1,14 @@
package claude
import (
"bytes"
"context"
"encoding/json"
"net"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"syscall"
"testing"
@@ -79,7 +81,7 @@ func TestReattachRestoresTasksAfterTheirStartWasTrimmed(t *testing.T) {
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2100, End: 2300, Data: json.RawMessage(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"finished-tool","content":"Command running in background with ID: old"}]}}`)})
select {
case <-notify:
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","tool_use_id":"native-bg","task_id":"task-17","status":"completed","summary":"Finished"}`)})
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","task_id":"task-17","status":"completed","summary":"Finished"}`)})
case <-stop:
return
}
@@ -160,3 +162,88 @@ func TestLiveUnresponsiveHostSurvivesAttachFailureWithoutDuplicateCLI(t *testing
t.Fatalf("a control failure spawned a replacement CLI: %d", len(spawns))
}
}
func TestNativeHostRetainsTaskIdentityBeyondSpoolCompactionAndRestart(t *testing.T) {
fixture := hostedSetup(t)
// Exercise the real host compactor rather than retaining all initiating
// events. The fake CLI supplies actual stream metadata, not test-side state.
hostConfig.Lock()
oldArgs := append([]string(nil), hostConfig.args...)
hostConfig.args = []string{"-spool-limit", "512"}
hostConfig.Unlock()
t.Cleanup(func() {
hostConfig.Lock()
hostConfig.args = oldArgs
hostConfig.Unlock()
})
script := filepath.Join(filepath.Dir(fixture.hostDir), "claude")
raw, err := os.ReadFile(script)
if err != nil {
t.Fatal(err)
}
raw = []byte(strings.Replace(string(raw), `"input": {"command": "sleep 600", "run_in_background": True}`, `"input": {"command": "sleep 600", "run_in_background": True, "description": "Release watcher"}`, 1))
raw = []byte(strings.Replace(string(raw), `send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}})`, `send({"type": "assistant", "timestamp": stamp(), "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}})`, 1))
if err := os.WriteFile(script, raw, 0o700); err != nil {
t.Fatal(err)
}
first := hostedChat(t)
session := newCreatedChat(t, first)
frames := subscribe(t, first, session.ID)
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
before, err := first.Shells(context.Background(), session.ID)
if err != nil || len(before) != 1 || before[0].StartedAt == 0 || before[0].Description != "Release watcher" {
t.Fatalf("native task metadata was not observed: %+v %v", before, err)
}
record := onlyRecord(t, fixture)
trimmed := false
for i := 0; i < 10; i++ {
if err := first.Prompt(context.Background(), session.ID, "hello"); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
spool, err := os.ReadFile(record.Spool)
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(spool, []byte("toolu_bg_fake")) {
trimmed = true
break
}
}
if !trimmed {
t.Fatal("fixture did not compact the task's initiating records")
}
first.Detach()
control, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
if err != nil {
t.Fatal(err)
}
_ = control.Close()
if status.Base <= 0 || status.Tasks == nil || len(*status.Tasks) != 1 {
t.Fatalf("task did not survive in authoritative host state: %+v", status)
}
second := hostedChat(t)
after, err := second.Shells(context.Background(), session.ID)
if err != nil || !reflect.DeepEqual(after, before) {
t.Fatalf("spool compaction/restart changed task identity or metadata:\nbefore=%+v\nafter=%+v\nerr=%v", before, after, err)
}
reused := onlyRecord(t, fixture)
if reused.HostPID != record.HostPID || reused.ClaudePID != record.ClaudePID {
t.Fatal("reattachment replaced the live host or CLI")
}
if calls := readArgs(t, fixture.logPath); len(calls) != 1 {
t.Fatalf("reattachment spawned another CLI: %d", len(calls))
}
frames = subscribe(t, second, session.ID)
if err := second.Prompt(context.Background(), session.ID, "notify me"); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
finished, err := second.Shells(context.Background(), session.ID)
if err != nil || len(finished) != 0 {
t.Fatalf("native completion did not settle the restored task: %+v %v", finished, err)
}
}