Retry retained Claude hosts from metadata readback without spawning work
This commit is contained in:
@@ -629,11 +629,13 @@ func (c *Chat) Answer(ctx context.Context, request protocol.ChatAnswerRequest) e
|
||||
|
||||
// Shells lists the commands still running in the session.
|
||||
func (c *Chat) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
|
||||
s := c.session(id)
|
||||
s, err := c.retainedSessionReadback(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s == nil {
|
||||
return []protocol.ChatShell{}, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.enrichShellStarts()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
+25
-18
@@ -181,11 +181,25 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e
|
||||
// is never killed to repair the control connection. Its registration blocks a
|
||||
// duplicate CLI until reconnect succeeds. Only dead references are removed.
|
||||
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
|
||||
return c.connectHostContext(context.Background(), record)
|
||||
}
|
||||
|
||||
func (c *Chat) connectHostContext(ctx context.Context, record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
|
||||
var conn *claudehost.Conn
|
||||
var status *claudehost.Status
|
||||
var err error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
conn, status, err = claudehost.Attach(record.Socket, 0, hostAttachTimeout)
|
||||
if ctx.Err() != nil {
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
timeout := hostAttachTimeout
|
||||
if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < timeout {
|
||||
timeout = time.Until(deadline)
|
||||
}
|
||||
if timeout <= 0 {
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
conn, status, err = claudehost.Attach(record.Socket, 0, timeout)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
@@ -196,7 +210,11 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn
|
||||
// A pre-timeout attach can still be draining when a paused host
|
||||
// resumes. Retry transient EOF/refusal without replacing its CLI.
|
||||
if attempt < 2 {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
select {
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
case <-ctx.Done():
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
@@ -376,7 +394,6 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
|
||||
frame, err := conn.Recv()
|
||||
if err != nil {
|
||||
goLive()
|
||||
p.markReady()
|
||||
if next, status := p.hostLost(conn, err); next != nil {
|
||||
conn = next
|
||||
plan = replayPlan{catchUpEnd: status.End, tasks: status.Tasks}
|
||||
@@ -626,11 +643,13 @@ func (c *Chat) ReleaseSession(_ context.Context, id string) error {
|
||||
// replay has rebuilt the session, so a turn that survived a restart is
|
||||
// observed and notifies.
|
||||
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
|
||||
s := c.session(id)
|
||||
s, err := c.retainedSessionReadback(ctx, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if s == nil {
|
||||
return false, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.running, nil
|
||||
@@ -639,17 +658,5 @@ func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
|
||||
// awaitReady waits, bounded, until a reattached process has replayed what it
|
||||
// missed, so approvals and the running turn are known before they are read.
|
||||
func (s *session) awaitReady(ctx context.Context) {
|
||||
s.mu.Lock()
|
||||
p := s.proc
|
||||
s.mu.Unlock()
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(10 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-p.ready:
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
_ = s.awaitReadbackReady(ctx)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
)
|
||||
|
||||
// retainedSessionReadback may reattach an existing verified live host after a
|
||||
// transient startup failure. It never spawns/resumes a CLI or sends a prompt.
|
||||
// Unavailable live work is unknown, not an authoritative empty task/turn set.
|
||||
func (c *Chat) retainedSessionReadback(ctx context.Context, id string) (*session, error) {
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.mu.Lock()
|
||||
closed := c.closed
|
||||
c.mu.Unlock()
|
||||
if closed {
|
||||
return nil, context.Canceled
|
||||
}
|
||||
s := c.session(id)
|
||||
if s != nil {
|
||||
s.mu.Lock()
|
||||
attached := s.proc != nil
|
||||
waiting := s.starting
|
||||
s.mu.Unlock()
|
||||
if attached {
|
||||
return s, s.awaitReadbackReady(ctx)
|
||||
}
|
||||
if waiting != nil {
|
||||
select {
|
||||
case <-waiting:
|
||||
continue
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
var retained *claudehost.Record
|
||||
if c.hostDir != "" {
|
||||
path := claudehost.PathsFor(c.hostDir, id).Record
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("read Claude Code host reference: %w", err)
|
||||
}
|
||||
if err == nil {
|
||||
var record claudehost.Record
|
||||
if json.Unmarshal(data, &record) != nil || record.SessionID != id || record.Socket == "" {
|
||||
return nil, fmt.Errorf("invalid retained Claude Code host reference for %s", id)
|
||||
}
|
||||
if hostReferenceLive(record) {
|
||||
retained = &record
|
||||
}
|
||||
}
|
||||
if retained == nil {
|
||||
// A hosted session without a live retained owner has no work
|
||||
// to reattach; do not reuse stale cached turn/task liveness.
|
||||
return nil, nil
|
||||
}
|
||||
} else {
|
||||
// Direct-mode callers retain their existing in-memory behavior.
|
||||
return s, nil
|
||||
}
|
||||
if s == nil {
|
||||
s = c.sessionFor(id, retained.Directory)
|
||||
}
|
||||
s.mu.Lock()
|
||||
if s.proc != nil || s.starting != nil {
|
||||
s.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
wait := make(chan struct{})
|
||||
s.starting = wait
|
||||
s.mu.Unlock()
|
||||
p, conn, plan := c.connectHostContext(ctx, *retained)
|
||||
c.mu.Lock()
|
||||
closed = c.closed
|
||||
c.mu.Unlock()
|
||||
if p != nil && (ctx.Err() != nil || closed) {
|
||||
_ = conn.Close()
|
||||
p = nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
if p != nil {
|
||||
s.proc = p
|
||||
s.adoptLocked(p)
|
||||
}
|
||||
s.starting = nil
|
||||
close(wait)
|
||||
s.mu.Unlock()
|
||||
if p == nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closed {
|
||||
return nil, context.Canceled
|
||||
}
|
||||
return nil, fmt.Errorf("Claude Code host for %s is unavailable; retained work was not stopped", id)
|
||||
}
|
||||
go p.readHost(conn, plan)
|
||||
return s, s.awaitReadbackReady(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// A replay deadline must propagate as unknown to startup recovery. Otherwise an
|
||||
// empty cache can incorrectly settle/delete the durable live-session journal.
|
||||
func (s *session) awaitReadbackReady(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
p := s.proc
|
||||
s.mu.Unlock()
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(10 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-p.ready:
|
||||
return nil
|
||||
case <-p.done:
|
||||
return fmt.Errorf("Claude Code host disconnected before replay completed")
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return fmt.Errorf("Claude Code host replay timeout: %w", context.DeadlineExceeded)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"reflect"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
)
|
||||
|
||||
func TestMetadataReadbackRetriesUnavailableStartupHostWithoutPrompt(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
before, err := first.Shells(context.Background(), session.ID)
|
||||
if err != nil || len(before) != 1 {
|
||||
t.Fatalf("fixture did not establish background work: %+v %v", before, err)
|
||||
}
|
||||
record := onlyRecord(t, fixture)
|
||||
first.Detach()
|
||||
if err := syscall.Kill(record.HostPID, syscall.SIGSTOP); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer syscall.Kill(record.HostPID, syscall.SIGCONT)
|
||||
// New's normal startup attach cannot reach the paused host. Read-only
|
||||
// recovery must not mistake that missing in-memory session for idle.
|
||||
after := hostedChat(t)
|
||||
if s := after.session(session.ID); s != nil {
|
||||
t.Fatal("paused host unexpectedly attached during startup")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
busy, probeErr := after.ProbeTurn(ctx, session.ID)
|
||||
cancel()
|
||||
if probeErr == nil {
|
||||
t.Fatalf("unavailable retained host reported authoritative turn state: %v", busy)
|
||||
}
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
shells, shellErr := after.Shells(ctx, session.ID)
|
||||
cancel()
|
||||
if shellErr == nil {
|
||||
t.Fatalf("unavailable retained host reported authoritative task state: %+v", shells)
|
||||
}
|
||||
if !alive(record.HostPID) || !alive(record.ClaudePID) {
|
||||
t.Fatal("metadata readback stopped live work")
|
||||
}
|
||||
if err := syscall.Kill(record.HostPID, syscall.SIGCONT); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "ProbeTurn alone to reattach the retained host", func() bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
busy, err := after.ProbeTurn(ctx, session.ID)
|
||||
if err != nil || busy {
|
||||
return false
|
||||
}
|
||||
s := after.session(session.ID)
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
attached := s.proc != nil
|
||||
s.mu.Unlock()
|
||||
return attached
|
||||
})
|
||||
shells, err = after.Shells(context.Background(), session.ID)
|
||||
if err != nil || !reflect.DeepEqual(shells, before) {
|
||||
t.Fatalf("metadata-only reattachment changed native tasks: before=%+v after=%+v err=%v", before, shells, err)
|
||||
}
|
||||
authoritative := onlyRecord(t, fixture)
|
||||
if record.HostPID != authoritative.HostPID || record.ClaudePID != authoritative.ClaudePID {
|
||||
t.Fatal("metadata-only recovery replaced the live processes")
|
||||
}
|
||||
if starts := readArgs(t, fixture.logPath); len(starts) != 1 {
|
||||
t.Fatalf("readback spawned/resumed a CLI: %d", len(starts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataReadbackReadinessDeadlineIsUnknown(t *testing.T) {
|
||||
chat := newChat(context.Background())
|
||||
defer chat.Close()
|
||||
s := chat.sessionFor("pending-replay", t.TempDir())
|
||||
p := newProcess(s, "")
|
||||
s.proc = p
|
||||
defer func() { s.proc = nil; p.finishDetached() }()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
_, err := chat.ProbeTurn(ctx, s.id)
|
||||
cancel()
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("replay timeout became authoritative turn state: %v", err)
|
||||
}
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
_, err = chat.Shells(ctx, s.id)
|
||||
cancel()
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("replay timeout became authoritative empty task set: %v", err)
|
||||
}
|
||||
p.markReady()
|
||||
if busy, err := chat.ProbeTurn(context.Background(), s.id); err != nil || busy {
|
||||
t.Fatalf("completed empty replay must be authoritative idle: %v %v", busy, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataReadbackMissingOrDeadHostIsIdle(t *testing.T) {
|
||||
hostedSetup(t)
|
||||
chat := newChat(context.Background())
|
||||
defer chat.Close()
|
||||
for _, id := range []string{"missing-host", "dead-host"} {
|
||||
if id == "dead-host" {
|
||||
paths := claudehost.PathsFor(chat.hostDir, id)
|
||||
if err := claudehost.EnsureDir(chat.hostDir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(claudehost.Record{SessionID: id, Socket: paths.Socket, HostPID: 0, ClaudePID: 0})
|
||||
if err := os.WriteFile(paths.Record, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if busy, err := chat.ProbeTurn(context.Background(), id); err != nil || busy {
|
||||
t.Fatalf("%s should be idle without a live owner: %v %v", id, busy, err)
|
||||
}
|
||||
if shells, err := chat.Shells(context.Background(), id); err != nil || len(shells) != 0 {
|
||||
t.Fatalf("%s should have no tasks without a live owner: %+v %v", id, shells, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user