Files
pipelines/internal/claude/hosted.go
T
alan c810437aa5 Claude Code chats survive server restarts in a detachable host
A pipelines-claude-host process owns each claude -p CLI and its pipes,
spools its stream-json output with offsets and serves a private Unix
socket. The server detaches on shutdown and reattaches on start,
replaying the spool to rebuild turns, approvals, questions, background
tasks and the live model. Explicit close, idle release and CLI exit stop
the host; a crashed host falls back to --resume with a notice.
2026-11-04 16:09:56 +00:00

592 lines
16 KiB
Go

package claude
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"git.a-wan.net/alan/pipelines/internal/claudehost"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Detachable hosts. With hosting configured, a session's CLI is owned by a
// pipelines-claude-host process (internal/claudehost) instead of this server,
// so a server restart does not end the conversation: the next server finds
// the host's registry file, reconnects, rebuilds the session's state from the
// spooled output and continues from its last acknowledged offset. Without
// hosting (tests, a missing host binary) the CLI is this server's child, as
// before, and a restart resumes it with --resume.
const hostAttachTimeout = 3 * time.Second
var hostConfig struct {
sync.Mutex
dir string
binary string
args []string
}
// ConfigureHosts makes Chats created afterwards run their CLIs in detachable
// hosts kept in dir. An empty dir turns hosting off. PIPELINES_CLAUDE_HOST
// names the host executable ("off" disables hosting); otherwise
// pipelines-claude-host is looked up next to this executable, then on PATH.
func ConfigureHosts(dir string) {
hostConfig.Lock()
defer hostConfig.Unlock()
hostConfig.dir = strings.TrimSpace(dir)
hostConfig.binary = ""
hostConfig.args = nil
if hostConfig.dir == "" {
return
}
binary := hostBinaryPath()
if binary == "" {
log.Printf("claude chat: pipelines-claude-host was not found; Claude Code chats stop on a server restart")
hostConfig.dir = ""
return
}
hostConfig.binary = binary
}
// DefaultHostDir is the private host directory for one server instance,
// scoped by its database path so two servers of one user never adopt each
// other's chats.
func DefaultHostDir(scope string) string {
base, err := os.UserCacheDir()
if err != nil || base == "" {
return ""
}
if absolute, err := filepath.Abs(scope); err == nil {
scope = absolute
}
sum := sha256.Sum256([]byte(scope))
return filepath.Join(base, "pipelines", "claude-hosts", hex.EncodeToString(sum[:6]))
}
func hostBinaryPath() string {
if value := strings.TrimSpace(os.Getenv("PIPELINES_CLAUDE_HOST")); value != "" {
if value == "off" {
return ""
}
return value
}
if executable, err := os.Executable(); err == nil {
candidate := filepath.Join(filepath.Dir(executable), "pipelines-claude-host")
if info, err := os.Stat(candidate); err == nil && !info.IsDir() {
return candidate
}
}
if path, err := exec.LookPath("pipelines-claude-host"); err == nil {
return path
}
return ""
}
// hostLink is a process's connection to its host.
type hostLink struct {
dir string
record claudehost.Record
mu sync.Mutex
conn *claudehost.Conn
committed int64
detached atomic.Bool
closing atomic.Bool
}
func (l *hostLink) current() *claudehost.Conn {
l.mu.Lock()
defer l.mu.Unlock()
return l.conn
}
func (l *hostLink) send(frame claudehost.Frame) error {
conn := l.current()
if conn == nil {
return errProcessClosed
}
if err := conn.Send(frame); err != nil {
return fmt.Errorf("write to the Claude Code host: %w", err)
}
return nil
}
// replayPlan says how a freshly attached reader treats output it has seen
// before: lines below silentUntil were processed by an earlier server and only
// rebuild state; control requests below catchUpEnd are applied only when the
// host still has them pending (answered ones must not reappear).
type replayPlan struct {
base int64
silentUntil int64
catchUpEnd int64
pending map[string]bool
busyAtAcked bool
restore bool
}
// startHosted starts a host for args and attaches to it.
func (s *session) startHosted(ctx context.Context, args []string, mode string, env []string) (*process, error) {
c := s.chat
hostConfig.Lock()
binary, hostArgs := hostConfig.binary, append([]string(nil), hostConfig.args...)
hostConfig.Unlock()
cmd, err := claudehost.Spawn(ctx, claudehost.SpawnOptions{
HostBinary: binary, HostArgs: hostArgs, Dir: c.hostDir, SessionID: s.id,
Claude: c.binary, Args: args, Env: env, Cwd: s.directory, Idle: idleProcessExpiry,
})
if err != nil {
return nil, err
}
// Reap the host if it ends while this server is still its parent.
go func() { _ = cmd.Wait() }()
paths := claudehost.PathsFor(c.hostDir, s.id)
conn, status, err := claudehost.Attach(paths.Socket, 0, hostAttachTimeout)
if err != nil {
claudehost.KillStale(c.hostDir, claudehost.Record{SessionID: s.id, HostPID: cmd.Process.Pid})
return nil, fmt.Errorf("attach to the Claude Code host: %w", err)
}
link := &hostLink{dir: c.hostDir, conn: conn, record: claudehost.Record{
SessionID: s.id, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: paths.Socket,
}}
p := newProcess(s, mode)
p.link = link
p.markReady()
go p.readHost(conn, replayPlan{})
return p, nil
}
// connectHost attaches to a host found in the registry and prepares a
// process for s without registering it. A host that is unreachable, belongs
// to another session or never finished initializing is stopped and its files
// removed; nil is returned and the next prompt starts a fresh CLI with
// --resume.
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
if err != nil {
log.Printf("claude chat %s: host unreachable (%v); the next prompt resumes the session", record.SessionID, err)
claudehost.KillStale(c.hostDir, record)
return nil, nil, replayPlan{}
}
var models []protocol.ChatModel
var account accountIdentity
if len(status.Init) > 0 {
models, account, err = parseInitialize(status.Init)
} else {
err = errors.New("the host's CLI never finished initializing")
}
if status.SessionID != record.SessionID || err != nil {
_ = conn.Close()
claudehost.KillStale(c.hostDir, record)
return nil, nil, replayPlan{}
}
directory := status.Directory
if directory == "" {
directory = record.Directory
}
s := c.sessionFor(record.SessionID, directory)
p := newProcess(s, status.Mode)
p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, record: claudehost.Record{
SessionID: record.SessionID, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: record.Socket,
}}
p.stderr.WriteString(status.Stderr)
p.models, p.account, p.appliedModel = models, account, status.Model
plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true}
for _, pending := range status.Pending {
plan.pending[pending.ID] = true
}
if status.Acked < status.Base {
// Output the previous server never processed was dropped by the spool
// bound; watchers reload the transcript.
log.Printf("claude chat %s: %d bytes of unread output exceeded the host spool", record.SessionID, status.Base-status.Acked)
plan.silentUntil = status.Base
}
return p, conn, plan
}
// adoptLocked applies an attached host's model and account to the session.
// Callers hold s.mu.
func (s *session) adoptLocked(p *process) {
s.models, s.account = p.models, p.account
s.appliedModel = p.appliedModel
if s.model == "" {
s.model = p.appliedModel
}
s.lastUsed = time.Now()
}
// attachHost adopts a live host found in the registry at server start.
func (c *Chat) attachHost(record claudehost.Record) bool {
p, conn, plan := c.connectHost(record)
if p == nil {
return false
}
s := p.session
s.mu.Lock()
if s.proc != nil || s.starting != nil {
s.mu.Unlock()
_ = conn.Close()
return false
}
s.proc = p
s.adoptLocked(p)
s.mu.Unlock()
go p.readHost(conn, plan)
return true
}
// reattachHosts adopts every live host of this server's directory. The
// registration is synchronous, so a prompt never races a second host for the
// same session; the replay that rebuilds each session runs in the background
// (see process.ready).
func (c *Chat) reattachHosts() {
if c.hostDir == "" {
return
}
var wg sync.WaitGroup
for _, record := range claudehost.Records(c.hostDir) {
wg.Add(1)
go func(record claudehost.Record) {
defer wg.Done()
if c.attachHost(record) {
log.Printf("claude chat %s: reattached to its running Claude Code host", record.SessionID)
}
}(record)
}
wg.Wait()
}
// existingHost adopts a host left for this session (one that started after
// this server's reattach pass) instead of starting a second one. The caller
// holds the session's starting slot and registers the process.
func (s *session) existingHost() *process {
c := s.chat
if c.hostDir == "" {
return nil
}
data, err := os.ReadFile(claudehost.PathsFor(c.hostDir, s.id).Record)
if err != nil {
return nil
}
var record claudehost.Record
if json.Unmarshal(data, &record) != nil || record.SessionID != s.id {
claudehost.Remove(c.hostDir, s.id)
return nil
}
p, conn, plan := c.connectHost(record)
if p == nil {
return nil
}
s.mu.Lock()
s.adoptLocked(p)
s.mu.Unlock()
go p.readHost(conn, plan)
return p
}
// readHost is the hosted counterpart of readStdout + wait.
func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
s := p.session
live := false
goLive := func() {
if live {
return
}
live = true
s.mu.Lock()
s.silent = false
turn := ""
if plan.restore {
if plan.busyAtAcked && !s.running {
turn = fmt.Sprintf("claude-%d", time.Now().UnixNano())
s.turnID = turn
s.running = true
s.steered = 0
} else if !plan.busyAtAcked {
// Output that starts a turn after the reconnect (a follow-up of
// messages written before it) is adopted as a running turn.
s.awaitFollowUp = true
}
}
s.mu.Unlock()
if turn != "" {
s.emit(protocol.ChatEventFrame{SessionID: s.id, NativeTurnID: turn, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: turn}})
}
if plan.restore {
// Watchers already attached reload the transcript.
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventResync})
}
}
if plan.silentUntil > plan.base {
s.mu.Lock()
s.silent = true
s.mu.Unlock()
} else {
goLive()
}
if plan.catchUpEnd <= plan.base {
p.markReady()
}
sinceAck := 0
for {
frame, err := conn.Recv()
if err != nil {
goLive()
p.markReady()
if next := p.hostLost(conn, err); next != nil {
conn = next
plan = replayPlan{}
continue
}
return
}
switch frame.Op {
case claudehost.OpOut:
if frame.Offset >= plan.silentUntil {
goLive()
}
if !(frame.Offset < plan.catchUpEnd && answeredRequest(frame.Data, plan.pending)) {
s.handleLine(p, frame.Data)
}
p.link.mu.Lock()
p.link.committed = frame.End
p.link.mu.Unlock()
sinceAck++
if sinceAck >= 64 || !conn.Buffered() {
_ = conn.Send(claudehost.Frame{Op: claudehost.OpAck, Offset: frame.End})
sinceAck = 0
}
if frame.End >= plan.silentUntil {
goLive()
}
if frame.End >= plan.catchUpEnd {
p.markReady()
}
case claudehost.OpExit:
goLive()
p.markReady()
if frame.Status != nil && frame.Status.Stderr != "" {
p.mu.Lock()
if p.stderr.Len() == 0 {
p.stderr.WriteString(frame.Status.Stderr)
}
p.mu.Unlock()
}
var exitErr error
if frame.Error != "" {
exitErr = errors.New(frame.Error)
}
p.link.closing.Store(true)
_ = conn.Close()
p.finish(exitErr)
return
case claudehost.OpClosed:
p.link.closing.Store(true)
}
}
}
// answeredRequest reports a replayed CLI control request that is no longer
// pending, so an approval already answered is not shown again.
func answeredRequest(line []byte, pending map[string]bool) bool {
var envelope struct {
Type string `json:"type"`
RequestID string `json:"request_id"`
}
if json.Unmarshal(line, &envelope) != nil || envelope.Type != "control_request" {
return false
}
return !pending[envelope.RequestID]
}
func (p *process) markReady() {
p.readyOnce.Do(func() { close(p.ready) })
}
// hostLost handles an ended host connection. A deliberate detach or close just
// ends the reader; an unexpected loss is retried once (the host may only have
// dropped the connection), and otherwise the host is treated as crashed: what
// it left is stopped and the turn is settled with a notice, and the next prompt
// resumes the session with --resume as after any release.
func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn {
link := p.link
_ = conn.Close()
if link.detached.Load() {
p.finishDetached()
return nil
}
if link.closing.Load() {
p.finish(nil)
return nil
}
link.mu.Lock()
committed := link.committed
link.mu.Unlock()
for attempt := 0; attempt < 2; attempt++ {
time.Sleep(150 * time.Millisecond)
if link.detached.Load() || link.closing.Load() {
break
}
next, _, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout)
if err == nil {
link.mu.Lock()
link.conn = next
link.mu.Unlock()
return next
}
}
if link.detached.Load() {
p.finishDetached()
return nil
}
if link.closing.Load() {
p.finish(nil)
return nil
}
log.Printf("claude chat %s: host connection lost (%v); falling back to --resume", p.session.id, cause)
claudehost.KillStale(link.dir, link.record)
p.mu.Lock()
p.stderr.Reset()
p.mu.Unlock()
p.finish(errors.New("the Claude Code host stopped unexpectedly; the next prompt resumes this session with --resume"))
return nil
}
// finishDetached ends a process this server let go of on shutdown. The CLI
// keeps running in its host; nothing is reported to watchers.
func (p *process) finishDetached() {
p.mu.Lock()
if p.exited {
p.mu.Unlock()
return
}
p.exited = true
pending := p.pending
p.pending = map[string]chan controlReply{}
close(p.done)
p.mu.Unlock()
for _, ch := range pending {
select {
case ch <- controlReply{err: "the server is shutting down"}:
default:
}
}
}
// closeHosted stops the host and its CLI (an explicit release).
func (p *process) closeHosted() {
link := p.link
link.closing.Store(true)
if err := link.send(claudehost.Frame{Op: claudehost.OpClose}); err != nil {
claudehost.KillStale(link.dir, link.record)
}
select {
case <-p.done:
case <-time.After(8 * time.Second):
claudehost.KillStale(link.dir, link.record)
if conn := link.current(); conn != nil {
_ = conn.Close()
}
}
}
// detach lets the host keep running while this server stops.
func (p *process) detach() {
link := p.link
if link == nil {
p.close()
return
}
link.detached.Store(true)
if conn := link.current(); conn != nil {
_ = conn.Close()
}
select {
case <-p.done:
case <-time.After(2 * time.Second):
}
}
// Detach disconnects from every hosted CLI without stopping it, for a server
// shutdown: the next server reattaches. CLIs this server owns directly (no
// host) are stopped as Close does.
func (c *Chat) Detach() {
c.cancel()
c.mu.Lock()
c.closed = true
sessions := make([]*session, 0, len(c.sessions))
for _, s := range c.sessions {
sessions = append(sessions, s)
}
c.sessions = map[string]*session{}
c.mu.Unlock()
var wg sync.WaitGroup
for _, s := range sessions {
s.mu.Lock()
p := s.proc
s.proc = nil
s.mu.Unlock()
if p == nil {
continue
}
wg.Add(1)
go func() {
defer wg.Done()
p.detach()
}()
}
wg.Wait()
}
// ReleaseSession stops a session's CLI (and its host) for an explicit operator
// close. The transcript stays; a later prompt resumes it.
func (c *Chat) ReleaseSession(_ context.Context, id string) error {
if s := c.session(id); s != nil {
s.closeProcess()
}
return nil
}
// ProbeTurn reports whether a turn is in flight, after a reattached host's
// replay has rebuilt the session, so a turn that survived a restart is
// observed and notifies.
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
s := c.session(id)
if s == nil {
return false, nil
}
s.awaitReady(ctx)
s.mu.Lock()
defer s.mu.Unlock()
return s.running, nil
}
// awaitReady waits, bounded, until a reattached process has replayed what it
// missed, so approvals and the running turn are known before they are read.
func (s *session) awaitReady(ctx context.Context) {
s.mu.Lock()
p := s.proc
s.mu.Unlock()
if p == nil {
return
}
timer := time.NewTimer(10 * time.Second)
defer timer.Stop()
select {
case <-p.ready:
case <-ctx.Done():
case <-timer.C:
}
}