Merge metadata-only startup reattachment and unknown-readback handling

This commit is contained in:
2026-11-07 16:37:01 +00:00
4 changed files with 299 additions and 20 deletions
+4 -2
View File
@@ -629,11 +629,13 @@ func (c *Chat) Answer(ctx context.Context, request protocol.ChatAnswerRequest) e
// Shells lists the commands still running in the session.
func (c *Chat) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) {
s := c.session(id)
s, err := c.retainedSessionReadback(ctx, id)
if err != nil {
return nil, err
}
if s == nil {
return []protocol.ChatShell{}, nil
}
s.awaitReady(ctx)
s.enrichShellStarts()
s.mu.Lock()
defer s.mu.Unlock()
+25 -18
View File
@@ -181,11 +181,25 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e
// is never killed to repair the control connection. Its registration blocks a
// duplicate CLI until reconnect succeeds. Only dead references are removed.
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
return c.connectHostContext(context.Background(), record)
}
func (c *Chat) connectHostContext(ctx context.Context, record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
var conn *claudehost.Conn
var status *claudehost.Status
var err error
for attempt := 0; attempt < 3; attempt++ {
conn, status, err = claudehost.Attach(record.Socket, 0, hostAttachTimeout)
if ctx.Err() != nil {
return nil, nil, replayPlan{}
}
timeout := hostAttachTimeout
if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < timeout {
timeout = time.Until(deadline)
}
if timeout <= 0 {
return nil, nil, replayPlan{}
}
conn, status, err = claudehost.Attach(record.Socket, 0, timeout)
if err == nil {
break
}
@@ -196,7 +210,11 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn
// A pre-timeout attach can still be draining when a paused host
// resumes. Retry transient EOF/refusal without replacing its CLI.
if attempt < 2 {
time.Sleep(100 * time.Millisecond)
select {
case <-time.After(100 * time.Millisecond):
case <-ctx.Done():
return nil, nil, replayPlan{}
}
}
}
if err != nil {
@@ -376,7 +394,6 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
frame, err := conn.Recv()
if err != nil {
goLive()
p.markReady()
if next, status := p.hostLost(conn, err); next != nil {
conn = next
plan = replayPlan{catchUpEnd: status.End, tasks: status.Tasks}
@@ -626,11 +643,13 @@ func (c *Chat) ReleaseSession(_ context.Context, id string) error {
// replay has rebuilt the session, so a turn that survived a restart is
// observed and notifies.
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
s := c.session(id)
s, err := c.retainedSessionReadback(ctx, id)
if err != nil {
return false, err
}
if s == nil {
return false, nil
}
s.awaitReady(ctx)
s.mu.Lock()
defer s.mu.Unlock()
return s.running, nil
@@ -639,17 +658,5 @@ func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
// awaitReady waits, bounded, until a reattached process has replayed what it
// missed, so approvals and the running turn are known before they are read.
func (s *session) awaitReady(ctx context.Context) {
s.mu.Lock()
p := s.proc
s.mu.Unlock()
if p == nil {
return
}
timer := time.NewTimer(10 * time.Second)
defer timer.Stop()
select {
case <-p.ready:
case <-ctx.Done():
case <-timer.C:
}
_ = s.awaitReadbackReady(ctx)
}
+135
View File
@@ -0,0 +1,135 @@
package claude
import (
"context"
"encoding/json"
"fmt"
"os"
"time"
"git.a-wan.net/alan/pipelines/internal/claudehost"
)
// retainedSessionReadback may reattach an existing verified live host after a
// transient startup failure. It never spawns/resumes a CLI or sends a prompt.
// Unavailable live work is unknown, not an authoritative empty task/turn set.
func (c *Chat) retainedSessionReadback(ctx context.Context, id string) (*session, error) {
for {
if err := ctx.Err(); err != nil {
return nil, err
}
c.mu.Lock()
closed := c.closed
c.mu.Unlock()
if closed {
return nil, context.Canceled
}
s := c.session(id)
if s != nil {
s.mu.Lock()
attached := s.proc != nil
waiting := s.starting
s.mu.Unlock()
if attached {
return s, s.awaitReadbackReady(ctx)
}
if waiting != nil {
select {
case <-waiting:
continue
case <-ctx.Done():
return nil, ctx.Err()
}
}
}
var retained *claudehost.Record
if c.hostDir != "" {
path := claudehost.PathsFor(c.hostDir, id).Record
data, err := os.ReadFile(path)
if err != nil && !os.IsNotExist(err) {
return nil, fmt.Errorf("read Claude Code host reference: %w", err)
}
if err == nil {
var record claudehost.Record
if json.Unmarshal(data, &record) != nil || record.SessionID != id || record.Socket == "" {
return nil, fmt.Errorf("invalid retained Claude Code host reference for %s", id)
}
if hostReferenceLive(record) {
retained = &record
}
}
if retained == nil {
// A hosted session without a live retained owner has no work
// to reattach; do not reuse stale cached turn/task liveness.
return nil, nil
}
} else {
// Direct-mode callers retain their existing in-memory behavior.
return s, nil
}
if s == nil {
s = c.sessionFor(id, retained.Directory)
}
s.mu.Lock()
if s.proc != nil || s.starting != nil {
s.mu.Unlock()
continue
}
wait := make(chan struct{})
s.starting = wait
s.mu.Unlock()
p, conn, plan := c.connectHostContext(ctx, *retained)
c.mu.Lock()
closed = c.closed
c.mu.Unlock()
if p != nil && (ctx.Err() != nil || closed) {
_ = conn.Close()
p = nil
}
s.mu.Lock()
if p != nil {
s.proc = p
s.adoptLocked(p)
}
s.starting = nil
close(wait)
s.mu.Unlock()
if p == nil {
if err := ctx.Err(); err != nil {
return nil, err
}
if closed {
return nil, context.Canceled
}
return nil, fmt.Errorf("Claude Code host for %s is unavailable; retained work was not stopped", id)
}
go p.readHost(conn, plan)
return s, s.awaitReadbackReady(ctx)
}
}
// A replay deadline must propagate as unknown to startup recovery. Otherwise an
// empty cache can incorrectly settle/delete the durable live-session journal.
func (s *session) awaitReadbackReady(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
s.mu.Lock()
p := s.proc
s.mu.Unlock()
if p == nil {
return nil
}
timer := time.NewTimer(10 * time.Second)
defer timer.Stop()
select {
case <-p.ready:
return nil
case <-p.done:
return fmt.Errorf("Claude Code host disconnected before replay completed")
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return fmt.Errorf("Claude Code host replay timeout: %w", context.DeadlineExceeded)
}
}
+135
View File
@@ -0,0 +1,135 @@
package claude
import (
"context"
"encoding/json"
"errors"
"os"
"reflect"
"syscall"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/claudehost"
)
func TestMetadataReadbackRetriesUnavailableStartupHostWithoutPrompt(t *testing.T) {
fixture := hostedSetup(t)
first := hostedChat(t)
session := newCreatedChat(t, first)
frames := subscribe(t, first, session.ID)
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
before, err := first.Shells(context.Background(), session.ID)
if err != nil || len(before) != 1 {
t.Fatalf("fixture did not establish background work: %+v %v", before, err)
}
record := onlyRecord(t, fixture)
first.Detach()
if err := syscall.Kill(record.HostPID, syscall.SIGSTOP); err != nil {
t.Fatal(err)
}
defer syscall.Kill(record.HostPID, syscall.SIGCONT)
// New's normal startup attach cannot reach the paused host. Read-only
// recovery must not mistake that missing in-memory session for idle.
after := hostedChat(t)
if s := after.session(session.ID); s != nil {
t.Fatal("paused host unexpectedly attached during startup")
}
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
busy, probeErr := after.ProbeTurn(ctx, session.ID)
cancel()
if probeErr == nil {
t.Fatalf("unavailable retained host reported authoritative turn state: %v", busy)
}
ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond)
shells, shellErr := after.Shells(ctx, session.ID)
cancel()
if shellErr == nil {
t.Fatalf("unavailable retained host reported authoritative task state: %+v", shells)
}
if !alive(record.HostPID) || !alive(record.ClaudePID) {
t.Fatal("metadata readback stopped live work")
}
if err := syscall.Kill(record.HostPID, syscall.SIGCONT); err != nil {
t.Fatal(err)
}
eventually(t, "ProbeTurn alone to reattach the retained host", func() bool {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
busy, err := after.ProbeTurn(ctx, session.ID)
if err != nil || busy {
return false
}
s := after.session(session.ID)
if s == nil {
return false
}
s.mu.Lock()
attached := s.proc != nil
s.mu.Unlock()
return attached
})
shells, err = after.Shells(context.Background(), session.ID)
if err != nil || !reflect.DeepEqual(shells, before) {
t.Fatalf("metadata-only reattachment changed native tasks: before=%+v after=%+v err=%v", before, shells, err)
}
authoritative := onlyRecord(t, fixture)
if record.HostPID != authoritative.HostPID || record.ClaudePID != authoritative.ClaudePID {
t.Fatal("metadata-only recovery replaced the live processes")
}
if starts := readArgs(t, fixture.logPath); len(starts) != 1 {
t.Fatalf("readback spawned/resumed a CLI: %d", len(starts))
}
}
func TestMetadataReadbackReadinessDeadlineIsUnknown(t *testing.T) {
chat := newChat(context.Background())
defer chat.Close()
s := chat.sessionFor("pending-replay", t.TempDir())
p := newProcess(s, "")
s.proc = p
defer func() { s.proc = nil; p.finishDetached() }()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
_, err := chat.ProbeTurn(ctx, s.id)
cancel()
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("replay timeout became authoritative turn state: %v", err)
}
ctx, cancel = context.WithTimeout(context.Background(), 20*time.Millisecond)
_, err = chat.Shells(ctx, s.id)
cancel()
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("replay timeout became authoritative empty task set: %v", err)
}
p.markReady()
if busy, err := chat.ProbeTurn(context.Background(), s.id); err != nil || busy {
t.Fatalf("completed empty replay must be authoritative idle: %v %v", busy, err)
}
}
func TestMetadataReadbackMissingOrDeadHostIsIdle(t *testing.T) {
hostedSetup(t)
chat := newChat(context.Background())
defer chat.Close()
for _, id := range []string{"missing-host", "dead-host"} {
if id == "dead-host" {
paths := claudehost.PathsFor(chat.hostDir, id)
if err := claudehost.EnsureDir(chat.hostDir); err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(claudehost.Record{SessionID: id, Socket: paths.Socket, HostPID: 0, ClaudePID: 0})
if err := os.WriteFile(paths.Record, raw, 0o600); err != nil {
t.Fatal(err)
}
}
if busy, err := chat.ProbeTurn(context.Background(), id); err != nil || busy {
t.Fatalf("%s should be idle without a live owner: %v %v", id, busy, err)
}
if shells, err := chat.Shells(context.Background(), id); err != nil || len(shells) != 0 {
t.Fatalf("%s should have no tasks without a live owner: %+v %v", id, shells, err)
}
}
}