Prove task recovery beyond spool trim and order live events after snapshots
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -180,7 +181,24 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e
|
||||
// is never killed to repair the control connection. Its registration blocks a
|
||||
// duplicate CLI until reconnect succeeds. Only dead references are removed.
|
||||
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
|
||||
conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
|
||||
var conn *claudehost.Conn
|
||||
var status *claudehost.Status
|
||||
var err error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
conn, status, err = claudehost.Attach(record.Socket, 0, hostAttachTimeout)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
var networkError net.Error
|
||||
if !hostReferenceLive(record) || (errors.As(err, &networkError) && networkError.Timeout()) {
|
||||
break
|
||||
}
|
||||
// A pre-timeout attach can still be draining when a paused host
|
||||
// resumes. Retry transient EOF/refusal without replacing its CLI.
|
||||
if attempt < 2 {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("claude chat %s: host unreachable (%v); retaining any live work for reconnect", record.SessionID, err)
|
||||
c.removeDeadHostReference(record)
|
||||
@@ -375,11 +393,16 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
|
||||
}
|
||||
switch frame.Op {
|
||||
case claudehost.OpOut:
|
||||
// Delivery can skip old records after spool compaction. A frame
|
||||
// starting at/after End is live: reconcile the older snapshot first.
|
||||
if frame.Offset >= plan.catchUpEnd {
|
||||
catchUpReady()
|
||||
}
|
||||
if frame.Offset >= plan.silentUntil {
|
||||
goLive()
|
||||
}
|
||||
if !(frame.Offset < plan.catchUpEnd && answeredRequest(frame.Data, plan.pending)) {
|
||||
s.handleLine(p, frame.Data)
|
||||
s.handleLine(p, s.resolveHostedTaskLine(frame.Data))
|
||||
}
|
||||
p.link.mu.Lock()
|
||||
p.link.committed = frame.End
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
@@ -68,3 +70,42 @@ func (s *session) reconcileHostTasks(tasks *[]claudehost.Task) {
|
||||
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventShell, Shell: &frames[i]})
|
||||
}
|
||||
}
|
||||
|
||||
// resolveHostedTaskLine correlates a completion that carries only the native
|
||||
// task id with the tool identity restored from the authoritative snapshot.
|
||||
// This is translation inside the server; no altered event is sent to the CLI.
|
||||
func (s *session) resolveHostedTaskLine(line []byte) []byte {
|
||||
if !bytes.Contains(line, []byte(`"task_notification"`)) {
|
||||
return line
|
||||
}
|
||||
var event struct {
|
||||
Subtype string `json:"subtype"`
|
||||
TaskID string `json:"task_id"`
|
||||
ToolUseID string `json:"tool_use_id"`
|
||||
}
|
||||
if json.Unmarshal(line, &event) != nil || event.Subtype != "task_notification" || event.TaskID == "" || event.ToolUseID != "" {
|
||||
return line
|
||||
}
|
||||
s.mu.Lock()
|
||||
id := ""
|
||||
for toolID, snapshot := range s.shells {
|
||||
if snapshot.taskID == event.TaskID {
|
||||
id = toolID
|
||||
break
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if id == "" {
|
||||
return line
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(line, &fields) != nil {
|
||||
return line
|
||||
}
|
||||
fields["tool_use_id"], _ = json.Marshal(id)
|
||||
normalized, err := json.Marshal(fields)
|
||||
if err != nil {
|
||||
return line
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
)
|
||||
|
||||
func TestLiveTaskEventAfterTrimmedReplayBoundaryWinsOverSnapshot(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
tasks []claudehost.Task
|
||||
line string
|
||||
want int
|
||||
}{
|
||||
{
|
||||
name: "new task remains running after older idle snapshot",
|
||||
tasks: []claudehost.Task{},
|
||||
line: `{"type":"assistant","message":{"id":"fresh","content":[{"type":"tool_use","id":"fresh-tool","name":"Bash","input":{"command":"new job","run_in_background":true}}]}}`,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "completion is not resurrected by older running snapshot",
|
||||
tasks: []claudehost.Task{{ToolUseID: "current-tool", TaskID: "native-task", Name: "Bash", Command: "existing job"}},
|
||||
line: `{"type":"system","subtype":"task_notification","task_id":"native-task","status":"completed","summary":"Finished"}`,
|
||||
want: 0,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
base, err := os.MkdirTemp("/tmp", "pcb")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(base)
|
||||
socket := filepath.Join(base, "host.sock")
|
||||
listener, err := net.Listen("unix", socket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
stop := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
var attach claudehost.Frame
|
||||
if json.NewDecoder(conn).Decode(&attach) != nil {
|
||||
return
|
||||
}
|
||||
out := json.NewEncoder(conn)
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpStatus, Status: &claudehost.Status{SessionID: "boundary-session", Base: 100, End: 200, Acked: 100, Init: json.RawMessage(`{"models":[{"value":"fixture"}]}`), Tasks: &tc.tasks}})
|
||||
// The old replay region was compacted before delivery. Its
|
||||
// first available record is newer than the snapshot boundary.
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 300, End: 400, Data: json.RawMessage(tc.line)})
|
||||
<-stop
|
||||
}()
|
||||
chat := newChat(context.Background())
|
||||
defer chat.Close()
|
||||
p, conn, plan := chat.connectHost(claudehost.Record{SessionID: "boundary-session", Socket: socket})
|
||||
if p == nil {
|
||||
close(stop)
|
||||
t.Fatal("fixture did not attach")
|
||||
}
|
||||
s := p.session
|
||||
s.mu.Lock()
|
||||
s.proc = p
|
||||
s.mu.Unlock()
|
||||
go p.readHost(conn, plan)
|
||||
defer func() { p.detach(); close(stop); <-done }()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
_, _ = chat.Shells(ctx, "boundary-session")
|
||||
p.link.mu.Lock()
|
||||
committed := p.link.committed
|
||||
p.link.mu.Unlock()
|
||||
// Shells can unblock just before the first live line completes.
|
||||
eventually(t, "the first live record to be committed", func() bool {
|
||||
p.link.mu.Lock()
|
||||
committed = p.link.committed
|
||||
p.link.mu.Unlock()
|
||||
return committed == 400
|
||||
})
|
||||
shells, err := chat.Shells(ctx, "boundary-session")
|
||||
if err != nil || len(shells) != tc.want {
|
||||
t.Fatalf("live record lost to stale snapshot: %+v %v", shells, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -79,7 +81,7 @@ func TestReattachRestoresTasksAfterTheirStartWasTrimmed(t *testing.T) {
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2100, End: 2300, Data: json.RawMessage(`{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"finished-tool","content":"Command running in background with ID: old"}]}}`)})
|
||||
select {
|
||||
case <-notify:
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","tool_use_id":"native-bg","task_id":"task-17","status":"completed","summary":"Finished"}`)})
|
||||
_ = out.Encode(claudehost.Frame{Op: claudehost.OpOut, Offset: 2300, End: 2400, Data: json.RawMessage(`{"type":"system","subtype":"task_notification","task_id":"task-17","status":"completed","summary":"Finished"}`)})
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
@@ -160,3 +162,88 @@ func TestLiveUnresponsiveHostSurvivesAttachFailureWithoutDuplicateCLI(t *testing
|
||||
t.Fatalf("a control failure spawned a replacement CLI: %d", len(spawns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeHostRetainsTaskIdentityBeyondSpoolCompactionAndRestart(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
// Exercise the real host compactor rather than retaining all initiating
|
||||
// events. The fake CLI supplies actual stream metadata, not test-side state.
|
||||
hostConfig.Lock()
|
||||
oldArgs := append([]string(nil), hostConfig.args...)
|
||||
hostConfig.args = []string{"-spool-limit", "512"}
|
||||
hostConfig.Unlock()
|
||||
t.Cleanup(func() {
|
||||
hostConfig.Lock()
|
||||
hostConfig.args = oldArgs
|
||||
hostConfig.Unlock()
|
||||
})
|
||||
script := filepath.Join(filepath.Dir(fixture.hostDir), "claude")
|
||||
raw, err := os.ReadFile(script)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw = []byte(strings.Replace(string(raw), `"input": {"command": "sleep 600", "run_in_background": True}`, `"input": {"command": "sleep 600", "run_in_background": True, "description": "Release watcher"}`, 1))
|
||||
raw = []byte(strings.Replace(string(raw), `send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}})`, `send({"type": "assistant", "timestamp": stamp(), "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}})`, 1))
|
||||
if err := os.WriteFile(script, raw, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
before, err := first.Shells(context.Background(), session.ID)
|
||||
if err != nil || len(before) != 1 || before[0].StartedAt == 0 || before[0].Description != "Release watcher" {
|
||||
t.Fatalf("native task metadata was not observed: %+v %v", before, err)
|
||||
}
|
||||
record := onlyRecord(t, fixture)
|
||||
trimmed := false
|
||||
for i := 0; i < 10; i++ {
|
||||
if err := first.Prompt(context.Background(), session.ID, "hello"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
spool, err := os.ReadFile(record.Spool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(spool, []byte("toolu_bg_fake")) {
|
||||
trimmed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !trimmed {
|
||||
t.Fatal("fixture did not compact the task's initiating records")
|
||||
}
|
||||
first.Detach()
|
||||
control, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = control.Close()
|
||||
if status.Base <= 0 || status.Tasks == nil || len(*status.Tasks) != 1 {
|
||||
t.Fatalf("task did not survive in authoritative host state: %+v", status)
|
||||
}
|
||||
second := hostedChat(t)
|
||||
after, err := second.Shells(context.Background(), session.ID)
|
||||
if err != nil || !reflect.DeepEqual(after, before) {
|
||||
t.Fatalf("spool compaction/restart changed task identity or metadata:\nbefore=%+v\nafter=%+v\nerr=%v", before, after, err)
|
||||
}
|
||||
reused := onlyRecord(t, fixture)
|
||||
if reused.HostPID != record.HostPID || reused.ClaudePID != record.ClaudePID {
|
||||
t.Fatal("reattachment replaced the live host or CLI")
|
||||
}
|
||||
if calls := readArgs(t, fixture.logPath); len(calls) != 1 {
|
||||
t.Fatalf("reattachment spawned another CLI: %d", len(calls))
|
||||
}
|
||||
frames = subscribe(t, second, session.ID)
|
||||
if err := second.Prompt(context.Background(), session.ID, "notify me"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
finished, err := second.Shells(context.Background(), session.ID)
|
||||
if err != nil || len(finished) != 0 {
|
||||
t.Fatalf("native completion did not settle the restored task: %+v %v", finished, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user