diff --git a/internal/claude/chat.go b/internal/claude/chat.go index 5ca86696..7416a059 100644 --- a/internal/claude/chat.go +++ b/internal/claude/chat.go @@ -629,11 +629,13 @@ func (c *Chat) Answer(ctx context.Context, request protocol.ChatAnswerRequest) e // Shells lists the commands still running in the session. func (c *Chat) Shells(ctx context.Context, id string) ([]protocol.ChatShell, error) { - s := c.session(id) + s, err := c.retainedSessionReadback(ctx, id) + if err != nil { + return nil, err + } if s == nil { return []protocol.ChatShell{}, nil } - s.awaitReady(ctx) s.enrichShellStarts() s.mu.Lock() defer s.mu.Unlock() diff --git a/internal/claude/hosted.go b/internal/claude/hosted.go index d7a83b55..2052d3b2 100644 --- a/internal/claude/hosted.go +++ b/internal/claude/hosted.go @@ -181,11 +181,25 @@ func (s *session) startHosted(ctx context.Context, args []string, mode string, e // is never killed to repair the control connection. Its registration blocks a // duplicate CLI until reconnect succeeds. Only dead references are removed. func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) { + return c.connectHostContext(context.Background(), record) +} + +func (c *Chat) connectHostContext(ctx context.Context, record claudehost.Record) (*process, *claudehost.Conn, replayPlan) { var conn *claudehost.Conn var status *claudehost.Status var err error for attempt := 0; attempt < 3; attempt++ { - conn, status, err = claudehost.Attach(record.Socket, 0, hostAttachTimeout) + if ctx.Err() != nil { + return nil, nil, replayPlan{} + } + timeout := hostAttachTimeout + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < timeout { + timeout = time.Until(deadline) + } + if timeout <= 0 { + return nil, nil, replayPlan{} + } + conn, status, err = claudehost.Attach(record.Socket, 0, timeout) if err == nil { break } @@ -196,7 +210,11 @@ func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn // A pre-timeout attach can still be draining when a paused host // resumes. Retry transient EOF/refusal without replacing its CLI. if attempt < 2 { - time.Sleep(100 * time.Millisecond) + select { + case <-time.After(100 * time.Millisecond): + case <-ctx.Done(): + return nil, nil, replayPlan{} + } } } if err != nil { @@ -376,7 +394,6 @@ func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { frame, err := conn.Recv() if err != nil { goLive() - p.markReady() if next, status := p.hostLost(conn, err); next != nil { conn = next plan = replayPlan{catchUpEnd: status.End, tasks: status.Tasks} @@ -626,11 +643,13 @@ func (c *Chat) ReleaseSession(_ context.Context, id string) error { // replay has rebuilt the session, so a turn that survived a restart is // observed and notifies. func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) { - s := c.session(id) + s, err := c.retainedSessionReadback(ctx, id) + if err != nil { + return false, err + } if s == nil { return false, nil } - s.awaitReady(ctx) s.mu.Lock() defer s.mu.Unlock() return s.running, nil @@ -639,17 +658,5 @@ func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) { // awaitReady waits, bounded, until a reattached process has replayed what it // missed, so approvals and the running turn are known before they are read. func (s *session) awaitReady(ctx context.Context) { - s.mu.Lock() - p := s.proc - s.mu.Unlock() - if p == nil { - return - } - timer := time.NewTimer(10 * time.Second) - defer timer.Stop() - select { - case <-p.ready: - case <-ctx.Done(): - case <-timer.C: - } + _ = s.awaitReadbackReady(ctx) } diff --git a/internal/claude/hosted_readback.go b/internal/claude/hosted_readback.go new file mode 100644 index 00000000..b6ab8c2d --- /dev/null +++ b/internal/claude/hosted_readback.go @@ -0,0 +1,135 @@ +package claude + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" +) + +// retainedSessionReadback may reattach an existing verified live host after a +// transient startup failure. It never spawns/resumes a CLI or sends a prompt. +// Unavailable live work is unknown, not an authoritative empty task/turn set. +func (c *Chat) retainedSessionReadback(ctx context.Context, id string) (*session, error) { + for { + if err := ctx.Err(); err != nil { + return nil, err + } + c.mu.Lock() + closed := c.closed + c.mu.Unlock() + if closed { + return nil, context.Canceled + } + s := c.session(id) + if s != nil { + s.mu.Lock() + attached := s.proc != nil + waiting := s.starting + s.mu.Unlock() + if attached { + return s, s.awaitReadbackReady(ctx) + } + if waiting != nil { + select { + case <-waiting: + continue + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } + var retained *claudehost.Record + if c.hostDir != "" { + path := claudehost.PathsFor(c.hostDir, id).Record + data, err := os.ReadFile(path) + if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("read Claude Code host reference: %w", err) + } + if err == nil { + var record claudehost.Record + if json.Unmarshal(data, &record) != nil || record.SessionID != id || record.Socket == "" { + return nil, fmt.Errorf("invalid retained Claude Code host reference for %s", id) + } + if hostReferenceLive(record) { + retained = &record + } + } + if retained == nil { + // A hosted session without a live retained owner has no work + // to reattach; do not reuse stale cached turn/task liveness. + return nil, nil + } + } else { + // Direct-mode callers retain their existing in-memory behavior. + return s, nil + } + if s == nil { + s = c.sessionFor(id, retained.Directory) + } + s.mu.Lock() + if s.proc != nil || s.starting != nil { + s.mu.Unlock() + continue + } + wait := make(chan struct{}) + s.starting = wait + s.mu.Unlock() + p, conn, plan := c.connectHostContext(ctx, *retained) + c.mu.Lock() + closed = c.closed + c.mu.Unlock() + if p != nil && (ctx.Err() != nil || closed) { + _ = conn.Close() + p = nil + } + s.mu.Lock() + if p != nil { + s.proc = p + s.adoptLocked(p) + } + s.starting = nil + close(wait) + s.mu.Unlock() + if p == nil { + if err := ctx.Err(); err != nil { + return nil, err + } + if closed { + return nil, context.Canceled + } + return nil, fmt.Errorf("Claude Code host for %s is unavailable; retained work was not stopped", id) + } + go p.readHost(conn, plan) + return s, s.awaitReadbackReady(ctx) + } +} + +// A replay deadline must propagate as unknown to startup recovery. Otherwise an +// empty cache can incorrectly settle/delete the durable live-session journal. +func (s *session) awaitReadbackReady(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + p := s.proc + s.mu.Unlock() + if p == nil { + return nil + } + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + select { + case <-p.ready: + return nil + case <-p.done: + return fmt.Errorf("Claude Code host disconnected before replay completed") + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return fmt.Errorf("Claude Code host replay timeout: %w", context.DeadlineExceeded) + } +} diff --git a/internal/claude/hosted_readback_test.go b/internal/claude/hosted_readback_test.go new file mode 100644 index 00000000..e323730f --- /dev/null +++ b/internal/claude/hosted_readback_test.go @@ -0,0 +1,135 @@ +package claude + +import ( + "context" + "encoding/json" + "errors" + "os" + "reflect" + "syscall" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" +) + +func TestMetadataReadbackRetriesUnavailableStartupHostWithoutPrompt(t *testing.T) { + fixture := hostedSetup(t) + first := hostedChat(t) + session := newCreatedChat(t, first) + frames := subscribe(t, first, session.ID) + if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + before, err := first.Shells(context.Background(), session.ID) + if err != nil || len(before) != 1 { + t.Fatalf("fixture did not establish background work: %+v %v", before, err) + } + record := onlyRecord(t, fixture) + first.Detach() + if err := syscall.Kill(record.HostPID, syscall.SIGSTOP); err != nil { + t.Fatal(err) + } + defer syscall.Kill(record.HostPID, syscall.SIGCONT) + // New's normal startup attach cannot reach the paused host. Read-only + // recovery must not mistake that missing in-memory session for idle. + after := hostedChat(t) + if s := after.session(session.ID); s != nil { + t.Fatal("paused host unexpectedly attached during startup") + } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + busy, probeErr := after.ProbeTurn(ctx, session.ID) + cancel() + if probeErr == nil { + t.Fatalf("unavailable retained host reported authoritative turn state: %v", busy) + } + ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond) + shells, shellErr := after.Shells(ctx, session.ID) + cancel() + if shellErr == nil { + t.Fatalf("unavailable retained host reported authoritative task state: %+v", shells) + } + if !alive(record.HostPID) || !alive(record.ClaudePID) { + t.Fatal("metadata readback stopped live work") + } + if err := syscall.Kill(record.HostPID, syscall.SIGCONT); err != nil { + t.Fatal(err) + } + eventually(t, "ProbeTurn alone to reattach the retained host", func() bool { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + busy, err := after.ProbeTurn(ctx, session.ID) + if err != nil || busy { + return false + } + s := after.session(session.ID) + if s == nil { + return false + } + s.mu.Lock() + attached := s.proc != nil + s.mu.Unlock() + return attached + }) + shells, err = after.Shells(context.Background(), session.ID) + if err != nil || !reflect.DeepEqual(shells, before) { + t.Fatalf("metadata-only reattachment changed native tasks: before=%+v after=%+v err=%v", before, shells, err) + } + authoritative := onlyRecord(t, fixture) + if record.HostPID != authoritative.HostPID || record.ClaudePID != authoritative.ClaudePID { + t.Fatal("metadata-only recovery replaced the live processes") + } + if starts := readArgs(t, fixture.logPath); len(starts) != 1 { + t.Fatalf("readback spawned/resumed a CLI: %d", len(starts)) + } +} + +func TestMetadataReadbackReadinessDeadlineIsUnknown(t *testing.T) { + chat := newChat(context.Background()) + defer chat.Close() + s := chat.sessionFor("pending-replay", t.TempDir()) + p := newProcess(s, "") + s.proc = p + defer func() { s.proc = nil; p.finishDetached() }() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + _, err := chat.ProbeTurn(ctx, s.id) + cancel() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("replay timeout became authoritative turn state: %v", err) + } + ctx, cancel = context.WithTimeout(context.Background(), 20*time.Millisecond) + _, err = chat.Shells(ctx, s.id) + cancel() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("replay timeout became authoritative empty task set: %v", err) + } + p.markReady() + if busy, err := chat.ProbeTurn(context.Background(), s.id); err != nil || busy { + t.Fatalf("completed empty replay must be authoritative idle: %v %v", busy, err) + } +} + +func TestMetadataReadbackMissingOrDeadHostIsIdle(t *testing.T) { + hostedSetup(t) + chat := newChat(context.Background()) + defer chat.Close() + for _, id := range []string{"missing-host", "dead-host"} { + if id == "dead-host" { + paths := claudehost.PathsFor(chat.hostDir, id) + if err := claudehost.EnsureDir(chat.hostDir); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(claudehost.Record{SessionID: id, Socket: paths.Socket, HostPID: 0, ClaudePID: 0}) + if err := os.WriteFile(paths.Record, raw, 0o600); err != nil { + t.Fatal(err) + } + } + if busy, err := chat.ProbeTurn(context.Background(), id); err != nil || busy { + t.Fatalf("%s should be idle without a live owner: %v %v", id, busy, err) + } + if shells, err := chat.Shells(context.Background(), id); err != nil || len(shells) != 0 { + t.Fatalf("%s should have no tasks without a live owner: %+v %v", id, shells, err) + } + } +}