Claude Code chats survive server restarts in a detachable host
A pipelines-claude-host process owns each claude -p CLI and its pipes, spools its stream-json output with offsets and serves a private Unix socket. The server detaches on shutdown and reattaches on start, replaying the spool to rebuild turns, approvals, questions, background tasks and the live model. Explicit close, idle release and CLI exit stop the host; a crashed host falls back to --resume with a notice.
This commit is contained in:
@@ -17,6 +17,7 @@ ARG PIPELINES_COMMIT=unknown
|
||||
ARG PIPELINES_DIRTY=unknown
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags "-X git.a-wan.net/alan/pipelines/internal/buildinfo.Version=${PIPELINES_VERSION} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Commit=${PIPELINES_COMMIT} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Dirty=${PIPELINES_DIRTY}" -o /out/pipelines-server ./cmd/pipelines-server
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags "-X git.a-wan.net/alan/pipelines/internal/buildinfo.Version=${PIPELINES_VERSION} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Commit=${PIPELINES_COMMIT} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Dirty=${PIPELINES_DIRTY}" -o /out/pipelines ./cmd/pipelines
|
||||
RUN CGO_ENABLED=0 go build -trimpath -o /out/pipelines-claude-host ./cmd/pipelines-claude-host
|
||||
|
||||
FROM ${OPENCODE_IMAGE}@${OPENCODE_DIGEST}
|
||||
ARG OPENCODE_VERSION
|
||||
@@ -42,6 +43,7 @@ RUN apk add --no-cache ca-certificates tzdata git curl nodejs npm python3 \
|
||||
&& chown -R pipelines:pipelines /data /workspace /home/pipelines
|
||||
COPY --from=build /out/pipelines-server /usr/local/bin/pipelines-server
|
||||
COPY --from=build /out/pipelines /usr/local/bin/pipelines
|
||||
COPY --from=build /out/pipelines-claude-host /usr/local/bin/pipelines-claude-host
|
||||
COPY --from=build /usr/local/go /usr/local/go
|
||||
COPY --chown=10001:10001 . /workspace
|
||||
ENV HOME=/home/pipelines \
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
// Command pipelines-claude-host keeps one Claude Code CLI process alive across
|
||||
// Pipelines server restarts. The server starts it; it is not run by hand. See
|
||||
// internal/claudehost and docs/chat-backends.md.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(claudehost.Main(os.Args[1:]))
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/buildinfo"
|
||||
"git.a-wan.net/alan/pipelines/internal/claude"
|
||||
"git.a-wan.net/alan/pipelines/internal/discovery"
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
"git.a-wan.net/alan/pipelines/internal/paths"
|
||||
@@ -131,6 +132,9 @@ func main() {
|
||||
options.OpenCodeStatus = openCodeStatus
|
||||
// One manager routes each chat to its owning harness. OpenCode remains the
|
||||
// default; Codex is attached when its app-server control socket is present.
|
||||
// Claude Code chats run in detachable hosts scoped to this database, so a
|
||||
// restart does not end them (internal/claudehost).
|
||||
claude.ConfigureHosts(claude.DefaultHostDir(*db))
|
||||
chats := server.NewChatManager(ctx, options.Chat, store)
|
||||
if len(chats.Backends()) > 0 {
|
||||
options.Chat = chats
|
||||
|
||||
+67
-3
@@ -43,9 +43,73 @@ credential or alter global settings. `PIPELINES_CLAUDE` may name the executable;
|
||||
otherwise `claude` is resolved from `PATH` and common per-user locations.
|
||||
|
||||
A new session pins a generated UUID with `--session-id`. When the process is
|
||||
released (idle or server restart), the next prompt resumes the same conversation
|
||||
with `--resume <id>`, so context and the selected model survive. Chat creation is
|
||||
local: the CLI starts on the first prompt or model-catalog read.
|
||||
released (idle, an explicit close, or a host that crashed), the next prompt
|
||||
resumes the same conversation with `--resume <id>`, so context and the selected
|
||||
model survive. Chat creation is local: the CLI starts on the first prompt or
|
||||
model-catalog read.
|
||||
|
||||
### Surviving a server restart
|
||||
|
||||
A server restart does not stop a Claude Code chat. The installed server runs each
|
||||
CLI inside `pipelines-claude-host` (`cmd/pipelines-claude-host`,
|
||||
`internal/claudehost`), a small helper installed next to `pipelines-server`. The
|
||||
host starts in its own session and process group, so neither launchd nor the
|
||||
supervisor stops it with the server (the systemd unit uses `KillMode=process`
|
||||
for the same reason). The host owns the CLI's stdin/stdout. It appends every
|
||||
output line to a per-session spool file with byte offsets and serves a Unix
|
||||
socket. The socket is mode 0600 in a 0700 directory:
|
||||
`<user cache dir>/pipelines/claude-hosts/<hash of the database path>/`. Two
|
||||
servers of one user never adopt each other's chats. Each host also writes a
|
||||
registry file (`<key>.json`) there.
|
||||
|
||||
The socket speaks one JSON object per line: `attach` (deliver output from an
|
||||
offset; a newer attach replaces the older connection), `in` (one stdin frame),
|
||||
`ack` (output processed up to an offset), `status` and `close`. Interrupts,
|
||||
model and mode switches travel as the CLI's own control requests inside `in`. The
|
||||
host snoops these frames to keep the CLI's initialize response (models, account),
|
||||
the applied model and permission mode, a turn's running state, and the control
|
||||
requests nobody has answered yet (`can_use_tool`, `AskUserQuestion`).
|
||||
|
||||
On shutdown the server only disconnects (`Detach`). On start it reads the
|
||||
registry and reattaches to every live host before serving. It replays the spool
|
||||
from its start without broadcasting, which rebuilds the session's stream state,
|
||||
background commands and agents, and pending approvals and questions. Requests
|
||||
already answered are skipped. Output after the last acknowledged offset, written
|
||||
while no server was attached, is then processed live. The turn is shown running
|
||||
if it was running at that offset, watchers get a resync, and the start-up turn
|
||||
probe keeps observing it for notifications. An approval or question asked before
|
||||
the restart can be answered after it. Stop works as usual, and the CLI is never
|
||||
restarted or initialized twice.
|
||||
|
||||
The host stops the CLI (EOF, then SIGINT, then SIGKILL to its process group) and
|
||||
removes its files only when one of these happens:
|
||||
|
||||
- The operator closes the chat (`chat_close`).
|
||||
- The server's idle release or live-process cap fires.
|
||||
- The CLI exits; the server then gets its stderr.
|
||||
- With no server attached, the CLI has been idle for the same 20 minutes, or
|
||||
no server has attached for 24 hours.
|
||||
|
||||
If the host itself dies, the server first tries to reconnect once. If that fails,
|
||||
it stops the orphaned CLI's process group (only when that pid's command line
|
||||
still names the session) and removes the host files. A running turn ends with
|
||||
the notice "the Claude Code host stopped unexpectedly; the next prompt resumes
|
||||
this session with --resume". The next prompt then resumes the session.
|
||||
|
||||
Limits:
|
||||
|
||||
- A container restart (Docker) still ends every host. So does a reboot or
|
||||
logout. The transcript survives and `--resume` applies.
|
||||
- The spool keeps at most 64 MiB of acknowledged output. Older output is
|
||||
dropped, so background-task state from before that point is not rebuilt after
|
||||
a restart. Unacknowledged output is dropped only beyond 256 MiB; the server
|
||||
then logs the gap and clients reload the transcript.
|
||||
- Turn bookkeeping that lived only in server memory restarts fresh: the turn id,
|
||||
and which mid-turn messages are still outstanding. Output that begins a
|
||||
follow-up turn after a reconnect is adopted as a running turn.
|
||||
- Without the host binary (`PIPELINES_CLAUDE_HOST=off`, or not found next to the
|
||||
server or on `PATH`), the CLI is the server's child as before and a restart
|
||||
resumes it.
|
||||
|
||||
Text and reasoning stream from the CLI's partial-message events; completed
|
||||
messages are de-duplicated against the streamed text. Tool calls appear as normal
|
||||
|
||||
@@ -325,6 +325,7 @@ func (s *session) ensureProcess(ctx context.Context) (*process, error) {
|
||||
p := s.proc
|
||||
s.lastUsed = time.Now()
|
||||
s.mu.Unlock()
|
||||
s.awaitReady(ctx)
|
||||
return p, nil
|
||||
}
|
||||
if s.starting != nil {
|
||||
@@ -408,6 +409,7 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
proc := s.proc
|
||||
running := s.running
|
||||
@@ -489,6 +491,7 @@ func (c *Chat) Permissions(ctx context.Context, id string) ([]protocol.ChatPermi
|
||||
if s == nil {
|
||||
return []protocol.ChatPermissionFrame{}, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
frames := make([]protocol.ChatPermissionFrame, 0, len(s.approvals))
|
||||
@@ -505,6 +508,7 @@ func (c *Chat) Forms(ctx context.Context, id string) ([]protocol.ChatForm, error
|
||||
if s == nil {
|
||||
return []protocol.ChatForm{}, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
forms := make([]protocol.ChatForm, 0, len(s.forms))
|
||||
@@ -629,6 +633,7 @@ func (c *Chat) Shells(ctx context.Context, id string) ([]protocol.ChatShell, err
|
||||
if s == nil {
|
||||
return []protocol.ChatShell{}, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
result := []protocol.ChatShell{}
|
||||
|
||||
@@ -4,9 +4,11 @@
|
||||
// mode: stream-json input carries user turns, stream-json output carries
|
||||
// assistant/tool events, and control requests carry interactive prompts
|
||||
// (can_use_tool) and session queries. This adapter owns one CLI process per
|
||||
// active session and mirrors that protocol into Pipelines chat frames. It never
|
||||
// changes the operator's Claude configuration, starts a daemon, or invents a
|
||||
// transcript: history is read back from the CLI's own session files.
|
||||
// active session and mirrors that protocol into Pipelines chat frames. In the
|
||||
// installed server each CLI runs inside a per-session pipelines-claude-host so
|
||||
// it survives a server restart (hosted.go). It never changes the operator's
|
||||
// Claude configuration or invents a transcript: history is read back from the
|
||||
// CLI's own session files.
|
||||
package claude
|
||||
|
||||
import (
|
||||
@@ -14,6 +16,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -22,6 +25,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/workenv"
|
||||
)
|
||||
@@ -96,6 +100,9 @@ type Chat struct {
|
||||
auth authEnv
|
||||
// dirEnv supplies each chat directory's own environment (direnv, mise).
|
||||
dirEnv *workenv.Resolver
|
||||
// hostDir is the private directory of detachable CLI hosts, "" when the
|
||||
// CLI is this server's child (see hosted.go).
|
||||
hostDir string
|
||||
// hinted records sessions whose registry directory has been resolved.
|
||||
hinted sync.Map
|
||||
|
||||
@@ -115,13 +122,30 @@ func New(ctx context.Context) *Chat {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
c := &Chat{ctx: ctx, cancel: cancel, binary: BinaryPath(), dirEnv: workenv.Default, sessions: map[string]*session{}}
|
||||
c := newChat(ctx)
|
||||
if c.hostDir != "" {
|
||||
if err := claudehost.EnsureDir(c.hostDir); err != nil {
|
||||
log.Printf("claude chat: host directory %s: %v; Claude Code chats stop on a server restart", c.hostDir, err)
|
||||
c.hostDir = ""
|
||||
}
|
||||
}
|
||||
c.reattachHosts()
|
||||
go c.reapIdle()
|
||||
return c
|
||||
}
|
||||
|
||||
// Close stops every owned CLI process. Session transcripts stay on disk.
|
||||
// newChat is New without the reattach pass and the reaper (tests).
|
||||
func newChat(ctx context.Context) *Chat {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
c := &Chat{ctx: ctx, cancel: cancel, binary: BinaryPath(), dirEnv: workenv.Default, sessions: map[string]*session{}}
|
||||
hostConfig.Lock()
|
||||
c.hostDir = hostConfig.dir
|
||||
hostConfig.Unlock()
|
||||
return c
|
||||
}
|
||||
|
||||
// Close stops every owned CLI process, hosted ones included. Session
|
||||
// transcripts stay on disk. A server shutdown uses Detach instead.
|
||||
func (c *Chat) Close() {
|
||||
c.cancel()
|
||||
c.mu.Lock()
|
||||
@@ -321,6 +345,10 @@ type session struct {
|
||||
// injectedSeen holds the identities of agent reports and notices already
|
||||
// published live, so a delivery the stream repeats is shown once.
|
||||
injectedSeen map[string]bool
|
||||
|
||||
// silent is set while a reattached host replays output an earlier server
|
||||
// already published: state is rebuilt, nothing is broadcast.
|
||||
silent bool
|
||||
}
|
||||
|
||||
type promptRecord struct {
|
||||
@@ -355,6 +383,10 @@ func (s *session) emit(frame protocol.ChatEventFrame) {
|
||||
s.compaction = nil
|
||||
}
|
||||
}
|
||||
if s.silent {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
targets := make([]chan protocol.ChatEventFrame, 0, len(s.subs))
|
||||
for ch := range s.subs {
|
||||
targets = append(targets, ch)
|
||||
|
||||
@@ -0,0 +1,591 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// Detachable hosts. With hosting configured, a session's CLI is owned by a
|
||||
// pipelines-claude-host process (internal/claudehost) instead of this server,
|
||||
// so a server restart does not end the conversation: the next server finds
|
||||
// the host's registry file, reconnects, rebuilds the session's state from the
|
||||
// spooled output and continues from its last acknowledged offset. Without
|
||||
// hosting (tests, a missing host binary) the CLI is this server's child, as
|
||||
// before, and a restart resumes it with --resume.
|
||||
|
||||
const hostAttachTimeout = 3 * time.Second
|
||||
|
||||
var hostConfig struct {
|
||||
sync.Mutex
|
||||
dir string
|
||||
binary string
|
||||
args []string
|
||||
}
|
||||
|
||||
// ConfigureHosts makes Chats created afterwards run their CLIs in detachable
|
||||
// hosts kept in dir. An empty dir turns hosting off. PIPELINES_CLAUDE_HOST
|
||||
// names the host executable ("off" disables hosting); otherwise
|
||||
// pipelines-claude-host is looked up next to this executable, then on PATH.
|
||||
func ConfigureHosts(dir string) {
|
||||
hostConfig.Lock()
|
||||
defer hostConfig.Unlock()
|
||||
hostConfig.dir = strings.TrimSpace(dir)
|
||||
hostConfig.binary = ""
|
||||
hostConfig.args = nil
|
||||
if hostConfig.dir == "" {
|
||||
return
|
||||
}
|
||||
binary := hostBinaryPath()
|
||||
if binary == "" {
|
||||
log.Printf("claude chat: pipelines-claude-host was not found; Claude Code chats stop on a server restart")
|
||||
hostConfig.dir = ""
|
||||
return
|
||||
}
|
||||
hostConfig.binary = binary
|
||||
}
|
||||
|
||||
// DefaultHostDir is the private host directory for one server instance,
|
||||
// scoped by its database path so two servers of one user never adopt each
|
||||
// other's chats.
|
||||
func DefaultHostDir(scope string) string {
|
||||
base, err := os.UserCacheDir()
|
||||
if err != nil || base == "" {
|
||||
return ""
|
||||
}
|
||||
if absolute, err := filepath.Abs(scope); err == nil {
|
||||
scope = absolute
|
||||
}
|
||||
sum := sha256.Sum256([]byte(scope))
|
||||
return filepath.Join(base, "pipelines", "claude-hosts", hex.EncodeToString(sum[:6]))
|
||||
}
|
||||
|
||||
func hostBinaryPath() string {
|
||||
if value := strings.TrimSpace(os.Getenv("PIPELINES_CLAUDE_HOST")); value != "" {
|
||||
if value == "off" {
|
||||
return ""
|
||||
}
|
||||
return value
|
||||
}
|
||||
if executable, err := os.Executable(); err == nil {
|
||||
candidate := filepath.Join(filepath.Dir(executable), "pipelines-claude-host")
|
||||
if info, err := os.Stat(candidate); err == nil && !info.IsDir() {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
if path, err := exec.LookPath("pipelines-claude-host"); err == nil {
|
||||
return path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hostLink is a process's connection to its host.
|
||||
type hostLink struct {
|
||||
dir string
|
||||
record claudehost.Record
|
||||
|
||||
mu sync.Mutex
|
||||
conn *claudehost.Conn
|
||||
committed int64
|
||||
|
||||
detached atomic.Bool
|
||||
closing atomic.Bool
|
||||
}
|
||||
|
||||
func (l *hostLink) current() *claudehost.Conn {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.conn
|
||||
}
|
||||
|
||||
func (l *hostLink) send(frame claudehost.Frame) error {
|
||||
conn := l.current()
|
||||
if conn == nil {
|
||||
return errProcessClosed
|
||||
}
|
||||
if err := conn.Send(frame); err != nil {
|
||||
return fmt.Errorf("write to the Claude Code host: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replayPlan says how a freshly attached reader treats output it has seen
|
||||
// before: lines below silentUntil were processed by an earlier server and only
|
||||
// rebuild state; control requests below catchUpEnd are applied only when the
|
||||
// host still has them pending (answered ones must not reappear).
|
||||
type replayPlan struct {
|
||||
base int64
|
||||
silentUntil int64
|
||||
catchUpEnd int64
|
||||
pending map[string]bool
|
||||
busyAtAcked bool
|
||||
restore bool
|
||||
}
|
||||
|
||||
// startHosted starts a host for args and attaches to it.
|
||||
func (s *session) startHosted(ctx context.Context, args []string, mode string, env []string) (*process, error) {
|
||||
c := s.chat
|
||||
hostConfig.Lock()
|
||||
binary, hostArgs := hostConfig.binary, append([]string(nil), hostConfig.args...)
|
||||
hostConfig.Unlock()
|
||||
cmd, err := claudehost.Spawn(ctx, claudehost.SpawnOptions{
|
||||
HostBinary: binary, HostArgs: hostArgs, Dir: c.hostDir, SessionID: s.id,
|
||||
Claude: c.binary, Args: args, Env: env, Cwd: s.directory, Idle: idleProcessExpiry,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Reap the host if it ends while this server is still its parent.
|
||||
go func() { _ = cmd.Wait() }()
|
||||
paths := claudehost.PathsFor(c.hostDir, s.id)
|
||||
conn, status, err := claudehost.Attach(paths.Socket, 0, hostAttachTimeout)
|
||||
if err != nil {
|
||||
claudehost.KillStale(c.hostDir, claudehost.Record{SessionID: s.id, HostPID: cmd.Process.Pid})
|
||||
return nil, fmt.Errorf("attach to the Claude Code host: %w", err)
|
||||
}
|
||||
link := &hostLink{dir: c.hostDir, conn: conn, record: claudehost.Record{
|
||||
SessionID: s.id, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: paths.Socket,
|
||||
}}
|
||||
p := newProcess(s, mode)
|
||||
p.link = link
|
||||
p.markReady()
|
||||
go p.readHost(conn, replayPlan{})
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// connectHost attaches to a host found in the registry and prepares a
|
||||
// process for s without registering it. A host that is unreachable, belongs
|
||||
// to another session or never finished initializing is stopped and its files
|
||||
// removed; nil is returned and the next prompt starts a fresh CLI with
|
||||
// --resume.
|
||||
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
|
||||
conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
|
||||
if err != nil {
|
||||
log.Printf("claude chat %s: host unreachable (%v); the next prompt resumes the session", record.SessionID, err)
|
||||
claudehost.KillStale(c.hostDir, record)
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
var models []protocol.ChatModel
|
||||
var account accountIdentity
|
||||
if len(status.Init) > 0 {
|
||||
models, account, err = parseInitialize(status.Init)
|
||||
} else {
|
||||
err = errors.New("the host's CLI never finished initializing")
|
||||
}
|
||||
if status.SessionID != record.SessionID || err != nil {
|
||||
_ = conn.Close()
|
||||
claudehost.KillStale(c.hostDir, record)
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
directory := status.Directory
|
||||
if directory == "" {
|
||||
directory = record.Directory
|
||||
}
|
||||
s := c.sessionFor(record.SessionID, directory)
|
||||
p := newProcess(s, status.Mode)
|
||||
p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, record: claudehost.Record{
|
||||
SessionID: record.SessionID, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: record.Socket,
|
||||
}}
|
||||
p.stderr.WriteString(status.Stderr)
|
||||
p.models, p.account, p.appliedModel = models, account, status.Model
|
||||
plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true}
|
||||
for _, pending := range status.Pending {
|
||||
plan.pending[pending.ID] = true
|
||||
}
|
||||
if status.Acked < status.Base {
|
||||
// Output the previous server never processed was dropped by the spool
|
||||
// bound; watchers reload the transcript.
|
||||
log.Printf("claude chat %s: %d bytes of unread output exceeded the host spool", record.SessionID, status.Base-status.Acked)
|
||||
plan.silentUntil = status.Base
|
||||
}
|
||||
return p, conn, plan
|
||||
}
|
||||
|
||||
// adoptLocked applies an attached host's model and account to the session.
|
||||
// Callers hold s.mu.
|
||||
func (s *session) adoptLocked(p *process) {
|
||||
s.models, s.account = p.models, p.account
|
||||
s.appliedModel = p.appliedModel
|
||||
if s.model == "" {
|
||||
s.model = p.appliedModel
|
||||
}
|
||||
s.lastUsed = time.Now()
|
||||
}
|
||||
|
||||
// attachHost adopts a live host found in the registry at server start.
|
||||
func (c *Chat) attachHost(record claudehost.Record) bool {
|
||||
p, conn, plan := c.connectHost(record)
|
||||
if p == nil {
|
||||
return false
|
||||
}
|
||||
s := p.session
|
||||
s.mu.Lock()
|
||||
if s.proc != nil || s.starting != nil {
|
||||
s.mu.Unlock()
|
||||
_ = conn.Close()
|
||||
return false
|
||||
}
|
||||
s.proc = p
|
||||
s.adoptLocked(p)
|
||||
s.mu.Unlock()
|
||||
go p.readHost(conn, plan)
|
||||
return true
|
||||
}
|
||||
|
||||
// reattachHosts adopts every live host of this server's directory. The
|
||||
// registration is synchronous, so a prompt never races a second host for the
|
||||
// same session; the replay that rebuilds each session runs in the background
|
||||
// (see process.ready).
|
||||
func (c *Chat) reattachHosts() {
|
||||
if c.hostDir == "" {
|
||||
return
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, record := range claudehost.Records(c.hostDir) {
|
||||
wg.Add(1)
|
||||
go func(record claudehost.Record) {
|
||||
defer wg.Done()
|
||||
if c.attachHost(record) {
|
||||
log.Printf("claude chat %s: reattached to its running Claude Code host", record.SessionID)
|
||||
}
|
||||
}(record)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// existingHost adopts a host left for this session (one that started after
|
||||
// this server's reattach pass) instead of starting a second one. The caller
|
||||
// holds the session's starting slot and registers the process.
|
||||
func (s *session) existingHost() *process {
|
||||
c := s.chat
|
||||
if c.hostDir == "" {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(claudehost.PathsFor(c.hostDir, s.id).Record)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var record claudehost.Record
|
||||
if json.Unmarshal(data, &record) != nil || record.SessionID != s.id {
|
||||
claudehost.Remove(c.hostDir, s.id)
|
||||
return nil
|
||||
}
|
||||
p, conn, plan := c.connectHost(record)
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.adoptLocked(p)
|
||||
s.mu.Unlock()
|
||||
go p.readHost(conn, plan)
|
||||
return p
|
||||
}
|
||||
|
||||
// readHost is the hosted counterpart of readStdout + wait.
|
||||
func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
|
||||
s := p.session
|
||||
live := false
|
||||
goLive := func() {
|
||||
if live {
|
||||
return
|
||||
}
|
||||
live = true
|
||||
s.mu.Lock()
|
||||
s.silent = false
|
||||
turn := ""
|
||||
if plan.restore {
|
||||
if plan.busyAtAcked && !s.running {
|
||||
turn = fmt.Sprintf("claude-%d", time.Now().UnixNano())
|
||||
s.turnID = turn
|
||||
s.running = true
|
||||
s.steered = 0
|
||||
} else if !plan.busyAtAcked {
|
||||
// Output that starts a turn after the reconnect (a follow-up of
|
||||
// messages written before it) is adopted as a running turn.
|
||||
s.awaitFollowUp = true
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if turn != "" {
|
||||
s.emit(protocol.ChatEventFrame{SessionID: s.id, NativeTurnID: turn, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: turn}})
|
||||
}
|
||||
if plan.restore {
|
||||
// Watchers already attached reload the transcript.
|
||||
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventResync})
|
||||
}
|
||||
}
|
||||
if plan.silentUntil > plan.base {
|
||||
s.mu.Lock()
|
||||
s.silent = true
|
||||
s.mu.Unlock()
|
||||
} else {
|
||||
goLive()
|
||||
}
|
||||
if plan.catchUpEnd <= plan.base {
|
||||
p.markReady()
|
||||
}
|
||||
sinceAck := 0
|
||||
for {
|
||||
frame, err := conn.Recv()
|
||||
if err != nil {
|
||||
goLive()
|
||||
p.markReady()
|
||||
if next := p.hostLost(conn, err); next != nil {
|
||||
conn = next
|
||||
plan = replayPlan{}
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
switch frame.Op {
|
||||
case claudehost.OpOut:
|
||||
if frame.Offset >= plan.silentUntil {
|
||||
goLive()
|
||||
}
|
||||
if !(frame.Offset < plan.catchUpEnd && answeredRequest(frame.Data, plan.pending)) {
|
||||
s.handleLine(p, frame.Data)
|
||||
}
|
||||
p.link.mu.Lock()
|
||||
p.link.committed = frame.End
|
||||
p.link.mu.Unlock()
|
||||
sinceAck++
|
||||
if sinceAck >= 64 || !conn.Buffered() {
|
||||
_ = conn.Send(claudehost.Frame{Op: claudehost.OpAck, Offset: frame.End})
|
||||
sinceAck = 0
|
||||
}
|
||||
if frame.End >= plan.silentUntil {
|
||||
goLive()
|
||||
}
|
||||
if frame.End >= plan.catchUpEnd {
|
||||
p.markReady()
|
||||
}
|
||||
case claudehost.OpExit:
|
||||
goLive()
|
||||
p.markReady()
|
||||
if frame.Status != nil && frame.Status.Stderr != "" {
|
||||
p.mu.Lock()
|
||||
if p.stderr.Len() == 0 {
|
||||
p.stderr.WriteString(frame.Status.Stderr)
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
var exitErr error
|
||||
if frame.Error != "" {
|
||||
exitErr = errors.New(frame.Error)
|
||||
}
|
||||
p.link.closing.Store(true)
|
||||
_ = conn.Close()
|
||||
p.finish(exitErr)
|
||||
return
|
||||
case claudehost.OpClosed:
|
||||
p.link.closing.Store(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// answeredRequest reports a replayed CLI control request that is no longer
|
||||
// pending, so an approval already answered is not shown again.
|
||||
func answeredRequest(line []byte, pending map[string]bool) bool {
|
||||
var envelope struct {
|
||||
Type string `json:"type"`
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
if json.Unmarshal(line, &envelope) != nil || envelope.Type != "control_request" {
|
||||
return false
|
||||
}
|
||||
return !pending[envelope.RequestID]
|
||||
}
|
||||
|
||||
func (p *process) markReady() {
|
||||
p.readyOnce.Do(func() { close(p.ready) })
|
||||
}
|
||||
|
||||
// hostLost handles an ended host connection. A deliberate detach or close just
|
||||
// ends the reader; an unexpected loss is retried once (the host may only have
|
||||
// dropped the connection), and otherwise the host is treated as crashed: what
|
||||
// it left is stopped and the turn is settled with a notice, and the next prompt
|
||||
// resumes the session with --resume as after any release.
|
||||
func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn {
|
||||
link := p.link
|
||||
_ = conn.Close()
|
||||
if link.detached.Load() {
|
||||
p.finishDetached()
|
||||
return nil
|
||||
}
|
||||
if link.closing.Load() {
|
||||
p.finish(nil)
|
||||
return nil
|
||||
}
|
||||
link.mu.Lock()
|
||||
committed := link.committed
|
||||
link.mu.Unlock()
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if link.detached.Load() || link.closing.Load() {
|
||||
break
|
||||
}
|
||||
next, _, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout)
|
||||
if err == nil {
|
||||
link.mu.Lock()
|
||||
link.conn = next
|
||||
link.mu.Unlock()
|
||||
return next
|
||||
}
|
||||
}
|
||||
if link.detached.Load() {
|
||||
p.finishDetached()
|
||||
return nil
|
||||
}
|
||||
if link.closing.Load() {
|
||||
p.finish(nil)
|
||||
return nil
|
||||
}
|
||||
log.Printf("claude chat %s: host connection lost (%v); falling back to --resume", p.session.id, cause)
|
||||
claudehost.KillStale(link.dir, link.record)
|
||||
p.mu.Lock()
|
||||
p.stderr.Reset()
|
||||
p.mu.Unlock()
|
||||
p.finish(errors.New("the Claude Code host stopped unexpectedly; the next prompt resumes this session with --resume"))
|
||||
return nil
|
||||
}
|
||||
|
||||
// finishDetached ends a process this server let go of on shutdown. The CLI
|
||||
// keeps running in its host; nothing is reported to watchers.
|
||||
func (p *process) finishDetached() {
|
||||
p.mu.Lock()
|
||||
if p.exited {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.exited = true
|
||||
pending := p.pending
|
||||
p.pending = map[string]chan controlReply{}
|
||||
close(p.done)
|
||||
p.mu.Unlock()
|
||||
for _, ch := range pending {
|
||||
select {
|
||||
case ch <- controlReply{err: "the server is shutting down"}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// closeHosted stops the host and its CLI (an explicit release).
|
||||
func (p *process) closeHosted() {
|
||||
link := p.link
|
||||
link.closing.Store(true)
|
||||
if err := link.send(claudehost.Frame{Op: claudehost.OpClose}); err != nil {
|
||||
claudehost.KillStale(link.dir, link.record)
|
||||
}
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(8 * time.Second):
|
||||
claudehost.KillStale(link.dir, link.record)
|
||||
if conn := link.current(); conn != nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// detach lets the host keep running while this server stops.
|
||||
func (p *process) detach() {
|
||||
link := p.link
|
||||
if link == nil {
|
||||
p.close()
|
||||
return
|
||||
}
|
||||
link.detached.Store(true)
|
||||
if conn := link.current(); conn != nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// Detach disconnects from every hosted CLI without stopping it, for a server
|
||||
// shutdown: the next server reattaches. CLIs this server owns directly (no
|
||||
// host) are stopped as Close does.
|
||||
func (c *Chat) Detach() {
|
||||
c.cancel()
|
||||
c.mu.Lock()
|
||||
c.closed = true
|
||||
sessions := make([]*session, 0, len(c.sessions))
|
||||
for _, s := range c.sessions {
|
||||
sessions = append(sessions, s)
|
||||
}
|
||||
c.sessions = map[string]*session{}
|
||||
c.mu.Unlock()
|
||||
var wg sync.WaitGroup
|
||||
for _, s := range sessions {
|
||||
s.mu.Lock()
|
||||
p := s.proc
|
||||
s.proc = nil
|
||||
s.mu.Unlock()
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
p.detach()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// ReleaseSession stops a session's CLI (and its host) for an explicit operator
|
||||
// close. The transcript stays; a later prompt resumes it.
|
||||
func (c *Chat) ReleaseSession(_ context.Context, id string) error {
|
||||
if s := c.session(id); s != nil {
|
||||
s.closeProcess()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProbeTurn reports whether a turn is in flight, after a reattached host's
|
||||
// replay has rebuilt the session, so a turn that survived a restart is
|
||||
// observed and notifies.
|
||||
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
|
||||
s := c.session(id)
|
||||
if s == nil {
|
||||
return false, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.running, nil
|
||||
}
|
||||
|
||||
// awaitReady waits, bounded, until a reattached process has replayed what it
|
||||
// missed, so approvals and the running turn are known before they are read.
|
||||
func (s *session) awaitReady(ctx context.Context) {
|
||||
s.mu.Lock()
|
||||
p := s.proc
|
||||
s.mu.Unlock()
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(10 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-p.ready:
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// The test binary doubles as pipelines-claude-host: a host spawned by a test
|
||||
// re-executes it with this variable set.
|
||||
const hostSelfTest = "PIPELINES_CLAUDE_HOST_SELFTEST"
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv(hostSelfTest) == "1" {
|
||||
os.Exit(claudehost.Main(os.Args[1:]))
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
type hostedFixture struct {
|
||||
hostDir string
|
||||
gate string
|
||||
logPath string
|
||||
}
|
||||
|
||||
// hostedSetup configures the fake CLI and detachable hosts in a short private
|
||||
// directory (Unix socket paths are length-limited).
|
||||
func hostedSetup(t *testing.T) hostedFixture {
|
||||
t.Helper()
|
||||
base, err := os.MkdirTemp("/tmp", "pch")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, err := os.ReadFile(filepath.Join("testdata", "fake-claude.py"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
script := filepath.Join(base, "claude")
|
||||
if err := os.WriteFile(script, source, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := hostedFixture{hostDir: filepath.Join(base, "h"), gate: filepath.Join(base, "gate"), logPath: filepath.Join(base, "args.log")}
|
||||
t.Setenv("PIPELINES_CLAUDE", script)
|
||||
t.Setenv("CLAUDE_CONFIG_DIR", filepath.Join(base, "config"))
|
||||
t.Setenv("FAKE_CLAUDE_LOG", fixture.logPath)
|
||||
t.Setenv("FAKE_CLAUDE_GATE", fixture.gate)
|
||||
t.Setenv("PIPELINES_CLAUDE_HOST", executable)
|
||||
t.Setenv(hostSelfTest, "1")
|
||||
ConfigureHosts(fixture.hostDir)
|
||||
t.Cleanup(func() {
|
||||
ConfigureHosts("")
|
||||
for _, record := range claudehost.Records(fixture.hostDir) {
|
||||
claudehost.KillStale(fixture.hostDir, record)
|
||||
}
|
||||
_ = os.RemoveAll(base)
|
||||
})
|
||||
return fixture
|
||||
}
|
||||
|
||||
func hostedChat(t *testing.T) *Chat {
|
||||
t.Helper()
|
||||
chat := New(context.Background())
|
||||
if chat.hostDir == "" {
|
||||
t.Fatal("hosting is not configured")
|
||||
}
|
||||
t.Cleanup(chat.Close)
|
||||
return chat
|
||||
}
|
||||
|
||||
func onlyRecord(t *testing.T, fixture hostedFixture) claudehost.Record {
|
||||
t.Helper()
|
||||
var records []claudehost.Record
|
||||
eventually(t, "a host registry record", func() bool {
|
||||
records = claudehost.Records(fixture.hostDir)
|
||||
return len(records) == 1
|
||||
})
|
||||
return records[0]
|
||||
}
|
||||
|
||||
func alive(pid int) bool { return pid > 0 && syscall.Kill(pid, 0) == nil }
|
||||
|
||||
func textFrame(contains string) func(protocol.ChatEventFrame) bool {
|
||||
return func(frame protocol.ChatEventFrame) bool {
|
||||
return strings.Contains(frame.Delta, contains) || (frame.Part != nil && strings.Contains(frame.Part.Text, contains))
|
||||
}
|
||||
}
|
||||
|
||||
func stepStarted(frame protocol.ChatEventFrame) bool {
|
||||
return frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepStarted
|
||||
}
|
||||
|
||||
// restart stands in for a server restart: the old Chat detaches (its host
|
||||
// keeps running) and a new one is built over the same host directory.
|
||||
func restart(t *testing.T, old *Chat) *Chat {
|
||||
t.Helper()
|
||||
old.Detach()
|
||||
return hostedChat(t)
|
||||
}
|
||||
|
||||
func TestHostedTurnSurvivesRestartAndReplaysMissedOutput(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "gate please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "text before the restart", textFrame("Working"))
|
||||
record := onlyRecord(t, fixture)
|
||||
first.Detach()
|
||||
if !alive(record.HostPID) || !alive(record.ClaudePID) {
|
||||
t.Fatal("detaching must leave the host and its CLI running")
|
||||
}
|
||||
|
||||
// The CLI finishes its turn while no server is attached.
|
||||
if err := os.WriteFile(fixture.gate, nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the turn result in the spool", func() bool {
|
||||
data, _ := os.ReadFile(record.Spool)
|
||||
return strings.Contains(string(data), `"type":"result"`)
|
||||
})
|
||||
|
||||
second := newChat(context.Background())
|
||||
t.Cleanup(second.Close)
|
||||
second.sessionFor(session.ID, session.Directory)
|
||||
frames = subscribe(t, second, session.ID)
|
||||
second.reattachHosts()
|
||||
waitFrame(t, frames, "the restored running turn", stepStarted)
|
||||
waitFrame(t, frames, "text produced while disconnected", textFrame("after the restart"))
|
||||
waitStepEnded(t, frames)
|
||||
if second.TurnRunning(session.ID) {
|
||||
t.Fatal("the replayed result must settle the turn")
|
||||
}
|
||||
if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 {
|
||||
t.Fatalf("the CLI must not be restarted, spawns: %+v", spawns)
|
||||
}
|
||||
// The same CLI keeps serving turns.
|
||||
if err := second.Prompt(context.Background(), session.ID, "hello again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the next turn's text", textFrame("Hello"))
|
||||
waitStepEnded(t, frames)
|
||||
}
|
||||
|
||||
func TestPendingApprovalIsAnsweredAfterRestart(t *testing.T) {
|
||||
hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
if err := first.Prompt(context.Background(), session.ID, "permission please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var before []protocol.ChatPermissionFrame
|
||||
eventually(t, "the approval before the restart", func() bool {
|
||||
before, _ = first.Permissions(context.Background(), session.ID)
|
||||
return len(before) == 1
|
||||
})
|
||||
second := restart(t, first)
|
||||
running, err := second.ProbeTurn(context.Background(), session.ID)
|
||||
if err != nil || !running {
|
||||
t.Fatalf("the restored turn must be running: %v %v", running, err)
|
||||
}
|
||||
after, err := second.Permissions(context.Background(), session.ID)
|
||||
if err != nil || len(after) != 1 || after[0].RequestID != before[0].RequestID {
|
||||
t.Fatalf("the pending approval must survive the restart: %+v %v", after, err)
|
||||
}
|
||||
frames := subscribe(t, second, session.ID)
|
||||
if err := second.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: after[0].RequestID, Decision: protocol.ChatDecisionOnce}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the approved tool result", func(frame protocol.ChatEventFrame) bool {
|
||||
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
|
||||
frame.Part.State.Status == protocol.ChatToolCompleted && len(frame.Part.State.Content) == 1 &&
|
||||
strings.Contains(frame.Part.State.Content[0].Text, "File created")
|
||||
})
|
||||
waitStepEnded(t, frames)
|
||||
|
||||
// An answered approval is not shown again by a later restart's replay.
|
||||
third := restart(t, second)
|
||||
if pending, _ := third.Permissions(context.Background(), session.ID); len(pending) != 0 {
|
||||
t.Fatalf("an answered approval reappeared: %+v", pending)
|
||||
}
|
||||
if third.TurnRunning(session.ID) {
|
||||
t.Fatal("a finished turn must not be restored as running")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuestionSurvivesRestart(t *testing.T) {
|
||||
hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
if err := first.Prompt(context.Background(), session.ID, "question please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the question before the restart", func() bool {
|
||||
forms, _ := first.Forms(context.Background(), session.ID)
|
||||
return len(forms) == 1
|
||||
})
|
||||
second := restart(t, first)
|
||||
forms, err := second.Forms(context.Background(), session.ID)
|
||||
if err != nil || len(forms) != 1 {
|
||||
t.Fatalf("the question must survive the restart: %+v %v", forms, err)
|
||||
}
|
||||
frames := subscribe(t, second, session.ID)
|
||||
if err := second.Answer(context.Background(), protocol.ChatAnswerRequest{SessionID: session.ID, FormID: forms[0].ID, Answers: map[string]any{"q0": "Beta"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the answered question", func(frame protocol.ChatEventFrame) bool {
|
||||
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
|
||||
len(frame.Part.State.Content) == 1 && strings.Contains(frame.Part.State.Content[0].Text, "Beta")
|
||||
})
|
||||
waitStepEnded(t, frames)
|
||||
}
|
||||
|
||||
func TestInterruptAfterRestart(t *testing.T) {
|
||||
hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "long task"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the long turn's text", textFrame("Hello"))
|
||||
second := restart(t, first)
|
||||
if running, _ := second.ProbeTurn(context.Background(), session.ID); !running {
|
||||
t.Fatal("the long turn must still be running after the restart")
|
||||
}
|
||||
frames = subscribe(t, second, session.ID)
|
||||
if err := second.Interrupt(context.Background(), session.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
if second.TurnRunning(session.ID) {
|
||||
t.Fatal("the interrupted turn must settle")
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelAndBackgroundTaskSurviveRestart(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
if _, err := first.Models(context.Background(), session.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := first.SwitchModel(context.Background(), session.ID, Backend, "haiku"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
if shells, _ := first.Shells(context.Background(), session.ID); len(shells) != 1 {
|
||||
t.Fatalf("the background command must be running: %+v", shells)
|
||||
}
|
||||
second := restart(t, first)
|
||||
shells, err := second.Shells(context.Background(), session.ID)
|
||||
if err != nil || len(shells) != 1 || shells[0].Command != "sleep 600" {
|
||||
t.Fatalf("the background command must still be listed after the restart: %+v %v", shells, err)
|
||||
}
|
||||
s := second.session(session.ID)
|
||||
s.mu.Lock()
|
||||
applied, models := s.appliedModel, len(s.models)
|
||||
s.mu.Unlock()
|
||||
if applied != "haiku" || models != 3 {
|
||||
t.Fatalf("the live model and catalog must be restored: %q %d", applied, models)
|
||||
}
|
||||
frames = subscribe(t, second, session.ID)
|
||||
if err := second.PromptWithMessageIDModel(context.Background(), session.ID, "notify me", "", "haiku"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
if shells, _ := second.Shells(context.Background(), session.ID); len(shells) != 0 {
|
||||
t.Fatalf("the notification must finish the restored command: %+v", shells)
|
||||
}
|
||||
if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 {
|
||||
t.Fatalf("the CLI must not be restarted, spawns: %d", len(spawns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostCrashFallsBackToResume(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
chat := hostedChat(t)
|
||||
session := newCreatedChat(t, chat)
|
||||
frames := subscribe(t, chat, session.ID)
|
||||
if err := chat.Prompt(context.Background(), session.ID, "long task"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the long turn's text", textFrame("Hello"))
|
||||
record := onlyRecord(t, fixture)
|
||||
if err := syscall.Kill(record.HostPID, syscall.SIGKILL); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the host crash notice", func(frame protocol.ChatEventFrame) bool {
|
||||
return frame.Part != nil && frame.Part.State != nil && frame.Part.State.Error != nil &&
|
||||
strings.Contains(frame.Part.State.Error.Message, "--resume")
|
||||
})
|
||||
waitStepEnded(t, frames)
|
||||
eventually(t, "the orphaned CLI to stop", func() bool { return !alive(record.ClaudePID) || isZombie(record.ClaudePID) })
|
||||
if _, err := os.Stat(record.Socket); !os.IsNotExist(err) {
|
||||
t.Fatalf("the crashed host's files must be removed: %v", err)
|
||||
}
|
||||
if err := chat.Prompt(context.Background(), session.ID, "hello again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
spawns := readArgs(t, fixture.logPath)
|
||||
if len(spawns) != 2 || argValue(spawns[1].Args, "--resume") != session.ID {
|
||||
t.Fatalf("the next prompt must resume the session: %+v", spawns)
|
||||
}
|
||||
}
|
||||
|
||||
func isZombie(pid int) bool {
|
||||
var status syscall.WaitStatus
|
||||
got, _ := syscall.Wait4(pid, &status, syscall.WNOHANG, nil)
|
||||
return got == pid
|
||||
}
|
||||
|
||||
func TestReleasingAHostedChatStopsItsHost(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
chat := hostedChat(t)
|
||||
session := newCreatedChat(t, chat)
|
||||
frames := subscribe(t, chat, session.ID)
|
||||
if err := chat.Prompt(context.Background(), session.ID, "hello"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
record := onlyRecord(t, fixture)
|
||||
if err := chat.ReleaseSession(context.Background(), session.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the host to exit", func() bool { return !alive(record.HostPID) })
|
||||
eventually(t, "the CLI to exit", func() bool { return !alive(record.ClaudePID) || isZombie(record.ClaudePID) })
|
||||
if records := claudehost.Records(fixture.hostDir); len(records) != 0 {
|
||||
t.Fatalf("a released host must remove its registry record: %+v", records)
|
||||
}
|
||||
// A restarted server finds nothing to reattach; the next prompt resumes.
|
||||
next := hostedChat(t)
|
||||
if next.session(session.ID) != nil {
|
||||
t.Fatal("a released session must not be reattached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostStatusReportsRegistryAndPrivateSocket(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
chat := hostedChat(t)
|
||||
session := newCreatedChat(t, chat)
|
||||
if _, err := chat.Models(context.Background(), session.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record := onlyRecord(t, fixture)
|
||||
info, err := os.Stat(record.Socket)
|
||||
if err != nil || info.Mode().Perm()&0o077 != 0 {
|
||||
t.Fatalf("the socket must be private: %v %v", info.Mode(), err)
|
||||
}
|
||||
dir, err := os.Stat(fixture.hostDir)
|
||||
if err != nil || dir.Mode().Perm() != 0o700 {
|
||||
t.Fatalf("the host directory must be private: %v %v", dir.Mode(), err)
|
||||
}
|
||||
conn, err := claudehost.Dial(record.Socket, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err := conn.Send(claudehost.Frame{Op: claudehost.OpStatus}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frame, err := conn.Recv()
|
||||
if err != nil || frame.Status == nil || frame.Status.SessionID != session.ID || len(frame.Status.Init) == 0 {
|
||||
raw, _ := json.Marshal(frame)
|
||||
t.Fatalf("status: %s %v", raw, err)
|
||||
}
|
||||
}
|
||||
+76
-22
@@ -14,6 +14,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
@@ -48,6 +49,23 @@ type process struct {
|
||||
// mode is the permission mode this process runs in, guarded by the
|
||||
// session's mu: set at spawn and after a confirmed set_permission_mode.
|
||||
mode string
|
||||
|
||||
// link is set when a detachable host owns the CLI (hosted.go); cmd and
|
||||
// stdin are then unused.
|
||||
link *hostLink
|
||||
// ready closes once a reattached process has replayed the output it
|
||||
// missed; a process this server started is ready at once.
|
||||
ready chan struct{}
|
||||
readyOnce sync.Once
|
||||
// models, account and appliedModel carry an attached host's
|
||||
// initialize state until the session adopts it.
|
||||
models []protocol.ChatModel
|
||||
account accountIdentity
|
||||
appliedModel string
|
||||
}
|
||||
|
||||
func newProcess(s *session, mode string) *process {
|
||||
return &process{session: s, pending: map[string]chan controlReply{}, done: make(chan struct{}), stderrDone: make(chan struct{}), ready: make(chan struct{}), mode: mode}
|
||||
}
|
||||
|
||||
type controlReply struct {
|
||||
@@ -97,6 +115,7 @@ func sessionDisplayName(title string) string {
|
||||
|
||||
// spawn starts the CLI in the session directory. A session whose transcript
|
||||
// already exists resumes it; a new one pins the id we recorded at create time.
|
||||
// With hosting configured the CLI runs in a detachable host (hosted.go).
|
||||
func (s *session) spawn(ctx context.Context) (*process, error) {
|
||||
if err := s.requireBinary(); err != nil {
|
||||
return nil, err
|
||||
@@ -107,34 +126,24 @@ func (s *session) spawn(ctx context.Context) (*process, error) {
|
||||
if info, err := os.Stat(s.directory); err != nil || !info.IsDir() {
|
||||
return nil, fmt.Errorf("Claude session directory is not available: %s", s.directory)
|
||||
}
|
||||
if p := s.existingHost(); p != nil {
|
||||
return p, nil
|
||||
}
|
||||
args, mode := s.cliArgs(findTranscriptFile(s.id) != "")
|
||||
cmd := exec.CommandContext(s.chat.ctx, s.chat.binary, args...)
|
||||
cmd.Dir = s.directory
|
||||
env, notices := s.chat.processEnv(ctx, s.directory)
|
||||
cmd.Env = env
|
||||
s.mu.Lock()
|
||||
s.envNotices = notices
|
||||
s.mu.Unlock()
|
||||
cmd.WaitDelay = 5 * time.Second
|
||||
stdin, err := cmd.StdinPipe()
|
||||
var p *process
|
||||
var err error
|
||||
if s.chat.hostDir != "" {
|
||||
p, err = s.startHosted(ctx, args, mode, env)
|
||||
} else {
|
||||
p, err = s.startDirect(args, mode, env)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start Claude Code: %w", err)
|
||||
}
|
||||
p := &process{session: s, cmd: cmd, stdin: stdin, pending: map[string]chan controlReply{}, done: make(chan struct{}), stderrDone: make(chan struct{}), mode: mode}
|
||||
go p.readStderr(stderr)
|
||||
go p.readStdout(stdout)
|
||||
go p.wait()
|
||||
// The CLI holds can_use_tool requests until the host has initialized. Do that
|
||||
// before any user turn so an approval is never silently denied.
|
||||
initCtx, cancel := context.WithTimeout(ctx, initializeTimeout)
|
||||
@@ -152,11 +161,47 @@ func (s *session) spawn(ctx context.Context) (*process, error) {
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// startDirect runs the CLI as this server's own child.
|
||||
func (s *session) startDirect(args []string, mode string, env []string) (*process, error) {
|
||||
cmd := exec.CommandContext(s.chat.ctx, s.chat.binary, args...)
|
||||
cmd.Dir = s.directory
|
||||
cmd.Env = env
|
||||
cmd.WaitDelay = 5 * time.Second
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start Claude Code: %w", err)
|
||||
}
|
||||
p := newProcess(s, mode)
|
||||
p.cmd, p.stdin = cmd, stdin
|
||||
p.markReady()
|
||||
go p.readStderr(stderr)
|
||||
go p.readStdout(stdout)
|
||||
go p.wait()
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (p *process) initialize(ctx context.Context) ([]protocol.ChatModel, accountIdentity, error) {
|
||||
raw, err := p.request(ctx, "initialize", map[string]any{"subtype": "initialize"})
|
||||
if err != nil {
|
||||
return nil, accountIdentity{}, fmt.Errorf("initialize Claude Code: %w%s", err, p.diagnostic())
|
||||
}
|
||||
return parseInitialize(raw)
|
||||
}
|
||||
|
||||
// parseInitialize reads the CLI's initialize response: its model catalog and
|
||||
// signed-in account.
|
||||
func parseInitialize(raw json.RawMessage) ([]protocol.ChatModel, accountIdentity, error) {
|
||||
var reply struct {
|
||||
Models []struct {
|
||||
Value string `json:"value"`
|
||||
@@ -237,7 +282,6 @@ func (p *process) write(value any) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
p.writeMu.Lock()
|
||||
defer p.writeMu.Unlock()
|
||||
p.mu.Lock()
|
||||
@@ -246,6 +290,10 @@ func (p *process) write(value any) error {
|
||||
if exited {
|
||||
return errProcessClosed
|
||||
}
|
||||
if p.link != nil {
|
||||
return p.link.send(claudehost.Frame{Op: claudehost.OpIn, Data: data})
|
||||
}
|
||||
data = append(data, '\n')
|
||||
if _, err := p.stdin.Write(data); err != nil {
|
||||
return fmt.Errorf("write to Claude Code: %w", err)
|
||||
}
|
||||
@@ -314,7 +362,9 @@ func (p *process) finish(err error) {
|
||||
default:
|
||||
}
|
||||
}
|
||||
_ = p.stdin.Close()
|
||||
if p.stdin != nil {
|
||||
_ = p.stdin.Close()
|
||||
}
|
||||
p.session.processExit(p, err)
|
||||
}
|
||||
|
||||
@@ -325,6 +375,10 @@ func (p *process) close() {
|
||||
if exited {
|
||||
return
|
||||
}
|
||||
if p.link != nil {
|
||||
p.closeHosted()
|
||||
return
|
||||
}
|
||||
_ = p.stdin.Close()
|
||||
if p.cmd.Process != nil {
|
||||
_ = p.cmd.Process.Signal(os.Interrupt)
|
||||
|
||||
+49
@@ -72,6 +72,15 @@ def user_message(text):
|
||||
send({"type": "system", "subtype": "init", "session_id": state["session"],
|
||||
"model": "claude-test", "permissionMode": "default", "cwd": os.getcwd()})
|
||||
message_id = "msg_fake_%d" % state["turn"]
|
||||
if "gate" in text:
|
||||
gated(message_id)
|
||||
return
|
||||
if "background" in text:
|
||||
background(message_id)
|
||||
return
|
||||
if "notify" in text:
|
||||
send({"type": "system", "subtype": "task_notification", "task_id": "b1",
|
||||
"tool_use_id": "toolu_bg_fake", "status": "completed", "summary": "Background command completed (exit code 0)"})
|
||||
send({"type": "stream_event", "event": {"type": "message_start", "message": {"id": message_id}}})
|
||||
send({"type": "stream_event", "event": {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}}})
|
||||
for delta in ("Hello ", "world"):
|
||||
@@ -125,6 +134,46 @@ def user_message(text):
|
||||
finish(message_id)
|
||||
|
||||
|
||||
def wait_for(path):
|
||||
import time
|
||||
while not os.path.exists(path):
|
||||
time.sleep(0.02)
|
||||
|
||||
|
||||
def gated(message_id):
|
||||
"""A turn that streams, then waits for FAKE_CLAUDE_GATE to exist before it
|
||||
streams the rest and finishes: the test restarts the server in between."""
|
||||
gate = os.environ["FAKE_CLAUDE_GATE"]
|
||||
send({"type": "stream_event", "event": {"type": "message_start", "message": {"id": message_id}}})
|
||||
send({"type": "stream_event", "event": {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}}})
|
||||
send({"type": "stream_event", "event": {"type": "content_block_delta", "index": 0,
|
||||
"delta": {"type": "text_delta", "text": "Working"}}})
|
||||
wait_for(gate)
|
||||
send({"type": "stream_event", "event": {"type": "content_block_delta", "index": 0,
|
||||
"delta": {"type": "text_delta", "text": " after the restart"}}})
|
||||
send({"type": "stream_event", "event": {"type": "content_block_stop", "index": 0}})
|
||||
send({"type": "stream_event", "event": {"type": "message_stop"}})
|
||||
text = "Working after the restart"
|
||||
append({"type": "assistant", "uuid": "asst-%d" % state["turn"], "timestamp": stamp(),
|
||||
"message": {"id": message_id, "role": "assistant", "model": "claude-test",
|
||||
"content": [{"type": "text", "text": text}]}})
|
||||
send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test",
|
||||
"content": [{"type": "text", "text": text}]}})
|
||||
finish(message_id)
|
||||
|
||||
|
||||
def background(message_id):
|
||||
"""Starts a background Bash command that a later "notify" turn finishes."""
|
||||
tool_id = "toolu_bg_fake"
|
||||
use = {"type": "tool_use", "id": tool_id, "name": "Bash",
|
||||
"input": {"command": "sleep 600", "run_in_background": True}}
|
||||
send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}})
|
||||
block = {"type": "tool_result", "tool_use_id": tool_id,
|
||||
"content": "Command running in background with ID: b1. Output is being written to: /tmp/b1.output"}
|
||||
send({"type": "user", "message": {"role": "user", "content": [block]}})
|
||||
finish(message_id)
|
||||
|
||||
|
||||
def finish(message_id):
|
||||
# Shape of SDKRateLimitEvent in Claude Code 2.1.316 (synthetic numbers).
|
||||
if os.environ.get("FAKE_CLAUDE_RATE_LIMIT"):
|
||||
|
||||
@@ -348,6 +348,9 @@ func (s *session) claimStreamedLocked(kind, full string, local int) (blockState,
|
||||
// broadcastLocked emits while the session lock is already held: the caller is a
|
||||
// stream-delta path that owns state it just updated.
|
||||
func (s *session) broadcastLocked(frame protocol.ChatEventFrame) {
|
||||
if s.silent {
|
||||
return
|
||||
}
|
||||
targets := make([]chan protocol.ChatEventFrame, 0, len(s.subs))
|
||||
for ch := range s.subs {
|
||||
targets = append(targets, ch)
|
||||
@@ -568,6 +571,9 @@ func (s *session) handleResult(envelope lineEnvelope) {
|
||||
turn := s.turnID
|
||||
running := s.running
|
||||
proc := s.proc
|
||||
if s.silent {
|
||||
proc = nil // replayed: the live usage query is made once caught up
|
||||
}
|
||||
s.running = false
|
||||
s.turnID = ""
|
||||
// Messages written mid-turn that the CLI did not fold into this turn run
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package claudehost
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Conn is a server's connection to one host.
|
||||
type Conn struct {
|
||||
conn net.Conn
|
||||
reader *bufio.Reader
|
||||
writeMu sync.Mutex
|
||||
}
|
||||
|
||||
// Dial connects to a host socket.
|
||||
func Dial(socket string, timeout time.Duration) (*Conn, error) {
|
||||
conn, err := net.DialTimeout("unix", socket, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Conn{conn: conn, reader: bufio.NewReaderSize(conn, 256<<10)}, nil
|
||||
}
|
||||
|
||||
// Send writes one frame.
|
||||
func (c *Conn) Send(frame Frame) error {
|
||||
data, err := json.Marshal(frame)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
c.writeMu.Lock()
|
||||
defer c.writeMu.Unlock()
|
||||
_, err = c.conn.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// Recv reads one frame.
|
||||
func (c *Conn) Recv() (Frame, error) {
|
||||
line, err := c.reader.ReadBytes('\n')
|
||||
if err != nil && len(strings.TrimSpace(string(line))) == 0 {
|
||||
return Frame{}, err
|
||||
}
|
||||
var frame Frame
|
||||
if err := json.Unmarshal(line, &frame); err != nil {
|
||||
return Frame{}, fmt.Errorf("invalid host frame: %w", err)
|
||||
}
|
||||
return frame, nil
|
||||
}
|
||||
|
||||
// Buffered reports whether more frames are already read from the socket.
|
||||
func (c *Conn) Buffered() bool { return c.reader.Buffered() > 0 }
|
||||
|
||||
func (c *Conn) Close() error { return c.conn.Close() }
|
||||
|
||||
// Attach dials socket and starts delivery from offset. It returns the status
|
||||
// the host replied with; output frames follow on the connection.
|
||||
func Attach(socket string, offset int64, timeout time.Duration) (*Conn, *Status, error) {
|
||||
conn, err := Dial(socket, timeout)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
_ = conn.conn.SetDeadline(time.Now().Add(timeout))
|
||||
if err := conn.Send(Frame{Op: OpAttach, Offset: offset}); err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
frame, err := conn.Recv()
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
_ = conn.conn.SetDeadline(time.Time{})
|
||||
if frame.Op != OpStatus || frame.Status == nil {
|
||||
conn.Close()
|
||||
return nil, nil, fmt.Errorf("unexpected host reply %q", frame.Op)
|
||||
}
|
||||
return conn, frame.Status, nil
|
||||
}
|
||||
|
||||
// SpawnOptions starts one host.
|
||||
type SpawnOptions struct {
|
||||
// HostBinary is the pipelines-claude-host executable.
|
||||
HostBinary string
|
||||
// HostArgs are extra arguments placed before the host flags (a test binary
|
||||
// uses them to select host mode).
|
||||
HostArgs []string
|
||||
Dir string
|
||||
SessionID string
|
||||
Claude string
|
||||
Args []string
|
||||
Env []string
|
||||
Cwd string
|
||||
Idle time.Duration
|
||||
}
|
||||
|
||||
// Spawn starts a host in its own session, so it outlives the caller, and
|
||||
// waits until its socket is serving. The caller reaps the returned command.
|
||||
func Spawn(ctx context.Context, options SpawnOptions) (*exec.Cmd, error) {
|
||||
if err := EnsureDir(options.Dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := PathsFor(options.Dir, options.SessionID)
|
||||
args := append([]string(nil), options.HostArgs...)
|
||||
args = append(args, "-dir", options.Dir, "-id", options.SessionID)
|
||||
if options.Idle > 0 {
|
||||
args = append(args, "-idle", options.Idle.String())
|
||||
}
|
||||
args = append(args, "--", options.Claude)
|
||||
args = append(args, options.Args...)
|
||||
cmd := exec.Command(options.HostBinary, args...)
|
||||
cmd.Dir = options.Cwd
|
||||
cmd.Env = options.Env
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
logFile, err := os.OpenFile(paths.Log, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer logFile.Close()
|
||||
cmd.Stderr = logFile
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start the Claude Code host: %w", err)
|
||||
}
|
||||
type ready struct {
|
||||
line string
|
||||
err error
|
||||
}
|
||||
result := make(chan ready, 1)
|
||||
go func() {
|
||||
line, err := bufio.NewReader(stdout).ReadString('\n')
|
||||
result <- ready{line: strings.TrimSpace(line), err: err}
|
||||
}()
|
||||
timer := time.NewTimer(15 * time.Second)
|
||||
defer timer.Stop()
|
||||
fail := func(err error) (*exec.Cmd, error) {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fail(ctx.Err())
|
||||
case <-timer.C:
|
||||
return fail(errors.New("the Claude Code host did not start in time"))
|
||||
case r := <-result:
|
||||
if strings.HasPrefix(r.line, "ready ") {
|
||||
return cmd, nil
|
||||
}
|
||||
message := strings.TrimPrefix(r.line, "error ")
|
||||
if message == "" {
|
||||
message = "the Claude Code host exited during start"
|
||||
if data, err := os.ReadFile(paths.Log); err == nil {
|
||||
if tail := strings.TrimSpace(string(data)); tail != "" {
|
||||
if len(tail) > 400 {
|
||||
tail = tail[len(tail)-400:]
|
||||
}
|
||||
message += ": " + tail
|
||||
}
|
||||
}
|
||||
}
|
||||
return fail(errors.New(message))
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessMatches reports whether pid is alive and its command line mentions
|
||||
// every needle, so a recorded pid that the system reused is never signalled.
|
||||
func ProcessMatches(pid int, needles ...string) bool {
|
||||
if pid <= 1 {
|
||||
return false
|
||||
}
|
||||
if syscall.Kill(pid, 0) != nil {
|
||||
return false
|
||||
}
|
||||
out, err := exec.Command("ps", "-o", "command=", "-p", strconv.Itoa(pid)).Output()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
command := string(out)
|
||||
for _, needle := range needles {
|
||||
if !strings.Contains(command, needle) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// KillStale stops what a dead or unreachable host left behind: the host
|
||||
// itself and its CLI's process group, each only when its command line still
|
||||
// names the session. It then removes the session's host files.
|
||||
func KillStale(dir string, record Record) {
|
||||
if ProcessMatches(record.HostPID, record.SessionID) {
|
||||
_ = syscall.Kill(record.HostPID, syscall.SIGKILL)
|
||||
}
|
||||
if ProcessMatches(record.ClaudePID, record.SessionID) {
|
||||
_ = syscall.Kill(-record.ClaudePID, syscall.SIGTERM)
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) && syscall.Kill(record.ClaudePID, 0) == nil {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
_ = syscall.Kill(-record.ClaudePID, syscall.SIGKILL)
|
||||
}
|
||||
Remove(dir, record.SessionID)
|
||||
}
|
||||
@@ -0,0 +1,710 @@
|
||||
package claudehost
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Defaults for a host started without explicit limits.
|
||||
const (
|
||||
DefaultIdle = 20 * time.Minute
|
||||
DefaultMaxDetached = 24 * time.Hour
|
||||
DefaultSpoolLimit = 64 << 20
|
||||
maxLineBytes = 32 << 20
|
||||
stderrRetained = 8 << 10
|
||||
)
|
||||
|
||||
type lineRef struct{ start, end int64 }
|
||||
|
||||
type transition struct {
|
||||
offset int64
|
||||
busy bool
|
||||
}
|
||||
|
||||
type client struct {
|
||||
conn net.Conn
|
||||
writeMu sync.Mutex
|
||||
gone chan struct{}
|
||||
once sync.Once
|
||||
exitSent bool
|
||||
}
|
||||
|
||||
func (c *client) send(frame Frame) error {
|
||||
data, err := json.Marshal(frame)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
c.writeMu.Lock()
|
||||
defer c.writeMu.Unlock()
|
||||
_, err = c.conn.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *client) close() {
|
||||
c.once.Do(func() {
|
||||
close(c.gone)
|
||||
_ = c.conn.Close()
|
||||
})
|
||||
}
|
||||
|
||||
type host struct {
|
||||
dir string
|
||||
id string
|
||||
paths Paths
|
||||
idle time.Duration
|
||||
maxDetached time.Duration
|
||||
spoolLimit int64
|
||||
listener net.Listener
|
||||
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
stdinMu sync.Mutex
|
||||
|
||||
fileMu sync.RWMutex // guards spool replacement against readers
|
||||
mu sync.Mutex
|
||||
spool *os.File
|
||||
base int64
|
||||
end int64
|
||||
lines []lineRef
|
||||
acked int64
|
||||
trans []transition
|
||||
busy bool
|
||||
// pending maps unanswered CLI control request ids to their offsets.
|
||||
pending map[string]int64
|
||||
initID string
|
||||
init json.RawMessage
|
||||
modelReqs map[string]string
|
||||
modeReqs map[string]string
|
||||
model string
|
||||
mode string
|
||||
exited bool
|
||||
exitErr string
|
||||
exitedAt time.Time
|
||||
stderr strings.Builder
|
||||
changed chan struct{}
|
||||
active *client
|
||||
detached time.Time
|
||||
activity time.Time
|
||||
stopping bool
|
||||
stopped chan struct{}
|
||||
claudeEnd chan struct{}
|
||||
}
|
||||
|
||||
// Main runs a host. args are the host flags, then "--", then the Claude Code
|
||||
// executable and its arguments. The working directory and environment are the
|
||||
// CLI's own. It returns the process exit code.
|
||||
func Main(args []string) int {
|
||||
flags := flag.NewFlagSet("pipelines-claude-host", flag.ContinueOnError)
|
||||
dir := flags.String("dir", "", "private host directory (registry, socket, spool)")
|
||||
id := flags.String("id", "", "Claude session id")
|
||||
idle := flags.Duration("idle", DefaultIdle, "release an idle CLI after this long with no server attached")
|
||||
maxDetached := flags.Duration("max-detached", DefaultMaxDetached, "release the CLI after this long with no server attached, even when busy")
|
||||
spoolLimit := flags.Int64("spool-limit", DefaultSpoolLimit, "bytes of acknowledged output kept in the spool")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
command := flags.Args()
|
||||
if len(command) == 0 || *id == "" || *dir == "" {
|
||||
fmt.Fprintln(os.Stderr, "usage: pipelines-claude-host -dir DIR -id SESSION -- claude ARGS...")
|
||||
return 2
|
||||
}
|
||||
log.SetFlags(log.LstdFlags)
|
||||
log.SetPrefix("claude-host " + *id + ": ")
|
||||
syscall.Umask(0o077)
|
||||
if err := EnsureDir(*dir); err != nil {
|
||||
log.Print(err)
|
||||
return 1
|
||||
}
|
||||
h := &host{
|
||||
dir: *dir, id: *id, paths: PathsFor(*dir, *id), idle: *idle, maxDetached: *maxDetached, spoolLimit: *spoolLimit,
|
||||
pending: map[string]int64{}, modelReqs: map[string]string{}, modeReqs: map[string]string{},
|
||||
changed: make(chan struct{}), stopped: make(chan struct{}), claudeEnd: make(chan struct{}),
|
||||
trans: []transition{{offset: 0, busy: false}}, activity: time.Now(), detached: time.Now(),
|
||||
}
|
||||
h.model, h.mode = flagValue(command, "--model"), flagValue(command, "--permission-mode")
|
||||
if err := h.start(command); err != nil {
|
||||
log.Print(err)
|
||||
fmt.Fprintln(os.Stdout, "error "+strings.ReplaceAll(err.Error(), "\n", " "))
|
||||
return 1
|
||||
}
|
||||
// The server waits for this line, then stops reading our stdout.
|
||||
fmt.Fprintln(os.Stdout, "ready "+h.paths.Socket)
|
||||
if null, err := os.OpenFile(os.DevNull, os.O_RDWR, 0); err == nil {
|
||||
_ = syscall.Dup2(int(null.Fd()), 1)
|
||||
_ = syscall.Dup2(int(null.Fd()), 0)
|
||||
_ = null.Close()
|
||||
}
|
||||
signals := make(chan os.Signal, 2)
|
||||
signal.Ignore(syscall.SIGHUP, syscall.SIGPIPE)
|
||||
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
||||
go func() {
|
||||
<-signals
|
||||
h.shutdown("signal")
|
||||
}()
|
||||
go h.accept()
|
||||
go h.watchIdle()
|
||||
<-h.stopped
|
||||
return 0
|
||||
}
|
||||
|
||||
func flagValue(args []string, name string) string {
|
||||
for index, arg := range args {
|
||||
if arg == name && index+1 < len(args) {
|
||||
return args[index+1]
|
||||
}
|
||||
if strings.HasPrefix(arg, name+"=") {
|
||||
return strings.TrimPrefix(arg, name+"=")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (h *host) start(command []string) error {
|
||||
if conn, err := net.DialTimeout("unix", h.paths.Socket, time.Second); err == nil {
|
||||
_ = conn.Close()
|
||||
return errors.New("a host is already serving this session")
|
||||
}
|
||||
_ = os.Remove(h.paths.Socket)
|
||||
spool, err := os.OpenFile(h.paths.Spool, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.spool = spool
|
||||
listener, err := net.Listen("unix", h.paths.Socket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Chmod(h.paths.Socket, 0o600)
|
||||
h.listener = listener
|
||||
cmd := exec.Command(command[0], command[1:]...)
|
||||
cmd.Env = os.Environ()
|
||||
// Its own process group, so the CLI and its tool subprocesses stop together.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = listener.Close()
|
||||
_ = os.Remove(h.paths.Socket)
|
||||
return fmt.Errorf("start Claude Code: %w", err)
|
||||
}
|
||||
h.cmd, h.stdin = cmd, stdin
|
||||
cwd, _ := os.Getwd()
|
||||
record := Record{Version: Version, SessionID: h.id, HostPID: os.Getpid(), ClaudePID: cmd.Process.Pid,
|
||||
Socket: h.paths.Socket, Spool: h.paths.Spool, Directory: cwd, Created: time.Now().UnixMilli()}
|
||||
data, _ := json.Marshal(record)
|
||||
if err := writeFileAtomic(h.paths.Record, data); err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
return err
|
||||
}
|
||||
stderrDone := make(chan struct{})
|
||||
go h.readStderr(stderr, stderrDone)
|
||||
readDone := make(chan struct{})
|
||||
go h.readStdout(stdout, readDone)
|
||||
go func() {
|
||||
err := cmd.Wait()
|
||||
<-readDone
|
||||
select {
|
||||
case <-stderrDone:
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.exited = true
|
||||
h.exitedAt = time.Now()
|
||||
if err != nil {
|
||||
h.exitErr = err.Error()
|
||||
}
|
||||
h.broadcastLocked()
|
||||
h.mu.Unlock()
|
||||
close(h.claudeEnd)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *host) readStderr(stderr io.Reader, done chan struct{}) {
|
||||
defer close(done)
|
||||
buffer := make([]byte, 4<<10)
|
||||
for {
|
||||
n, err := stderr.Read(buffer)
|
||||
if n > 0 {
|
||||
h.mu.Lock()
|
||||
if remaining := stderrRetained - h.stderr.Len(); remaining > 0 {
|
||||
if n > remaining {
|
||||
n = remaining
|
||||
}
|
||||
h.stderr.Write(buffer[:n])
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *host) readStdout(stdout io.Reader, done chan struct{}) {
|
||||
defer close(done)
|
||||
reader := bufio.NewReaderSize(stdout, 64<<10)
|
||||
for {
|
||||
line, err := reader.ReadBytes('\n')
|
||||
if len(line) > 0 && len(line) <= maxLineBytes {
|
||||
trimmed := strings.TrimSpace(string(line))
|
||||
if trimmed != "" {
|
||||
h.appendLine([]byte(trimmed))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// broadcastLocked wakes every sender. Callers hold h.mu.
|
||||
func (h *host) broadcastLocked() {
|
||||
close(h.changed)
|
||||
h.changed = make(chan struct{})
|
||||
}
|
||||
|
||||
type snoop struct {
|
||||
Type string `json:"type"`
|
||||
Subtype string `json:"subtype"`
|
||||
RequestID string `json:"request_id"`
|
||||
ParentToolUseID string `json:"parent_tool_use_id"`
|
||||
Request struct {
|
||||
Subtype string `json:"subtype"`
|
||||
Model string `json:"model"`
|
||||
Mode string `json:"mode"`
|
||||
} `json:"request"`
|
||||
Response struct {
|
||||
Subtype string `json:"subtype"`
|
||||
RequestID string `json:"request_id"`
|
||||
Response json.RawMessage `json:"response"`
|
||||
} `json:"response"`
|
||||
}
|
||||
|
||||
func (h *host) setBusyLocked(busy bool, offset int64) {
|
||||
if h.busy == busy {
|
||||
return
|
||||
}
|
||||
h.busy = busy
|
||||
h.trans = append(h.trans, transition{offset: offset, busy: busy})
|
||||
}
|
||||
|
||||
func (h *host) busyAtLocked(offset int64) bool {
|
||||
busy := false
|
||||
for _, t := range h.trans {
|
||||
if t.offset > offset {
|
||||
break
|
||||
}
|
||||
busy = t.busy
|
||||
}
|
||||
return busy
|
||||
}
|
||||
|
||||
func (h *host) appendLine(line []byte) {
|
||||
var env snoop
|
||||
_ = json.Unmarshal(line, &env)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
start := h.end
|
||||
data := append(append([]byte(nil), line...), '\n')
|
||||
h.fileMu.RLock()
|
||||
_, err := h.spool.WriteAt(data, start-h.base)
|
||||
h.fileMu.RUnlock()
|
||||
if err != nil {
|
||||
log.Printf("spool write: %v", err)
|
||||
}
|
||||
h.end += int64(len(data))
|
||||
h.lines = append(h.lines, lineRef{start: start, end: h.end})
|
||||
h.activity = time.Now()
|
||||
switch env.Type {
|
||||
case "control_request":
|
||||
if env.RequestID != "" {
|
||||
h.pending[env.RequestID] = start
|
||||
}
|
||||
case "control_cancel_request":
|
||||
delete(h.pending, env.RequestID)
|
||||
case "control_response":
|
||||
id := env.Response.RequestID
|
||||
if env.Response.Subtype == "success" {
|
||||
if id != "" && id == h.initID {
|
||||
h.init = append(json.RawMessage(nil), env.Response.Response...)
|
||||
}
|
||||
if model, ok := h.modelReqs[id]; ok {
|
||||
h.model = model
|
||||
}
|
||||
if mode, ok := h.modeReqs[id]; ok {
|
||||
h.mode = mode
|
||||
}
|
||||
}
|
||||
delete(h.modelReqs, id)
|
||||
delete(h.modeReqs, id)
|
||||
case "result":
|
||||
h.setBusyLocked(false, h.end)
|
||||
// A finished turn no longer waits on any approval.
|
||||
h.pending = map[string]int64{}
|
||||
case "stream_event", "assistant":
|
||||
if env.ParentToolUseID == "" {
|
||||
h.setBusyLocked(true, start)
|
||||
}
|
||||
}
|
||||
h.compactLocked()
|
||||
h.broadcastLocked()
|
||||
}
|
||||
|
||||
// compactLocked bounds the spool. Acknowledged output beyond the limit is
|
||||
// dropped first; unacknowledged output is dropped only beyond four times the
|
||||
// limit (a server gone for very long), which a reconnecting server sees as a
|
||||
// gap because Acked < Base.
|
||||
func (h *host) compactLocked() {
|
||||
size := h.end - h.base
|
||||
if size <= h.spoolLimit {
|
||||
return
|
||||
}
|
||||
keep := h.acked
|
||||
if h.end-keep > 4*h.spoolLimit {
|
||||
keep = h.end - h.spoolLimit
|
||||
}
|
||||
index := sort.Search(len(h.lines), func(i int) bool { return h.lines[i].start >= keep })
|
||||
if index == 0 || index >= len(h.lines) {
|
||||
return
|
||||
}
|
||||
keep = h.lines[index].start
|
||||
if keep-h.base < size/4 {
|
||||
return // not worth rewriting yet
|
||||
}
|
||||
next, err := os.OpenFile(h.paths.Spool+".next", os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
h.fileMu.Lock()
|
||||
defer h.fileMu.Unlock()
|
||||
if _, err := io.Copy(next, io.NewSectionReader(h.spool, keep-h.base, h.end-keep)); err != nil {
|
||||
_ = next.Close()
|
||||
_ = os.Remove(h.paths.Spool + ".next")
|
||||
return
|
||||
}
|
||||
if err := os.Rename(h.paths.Spool+".next", h.paths.Spool); err != nil {
|
||||
_ = next.Close()
|
||||
return
|
||||
}
|
||||
_ = h.spool.Close()
|
||||
h.spool = next
|
||||
h.base = keep
|
||||
h.lines = append([]lineRef(nil), h.lines[index:]...)
|
||||
}
|
||||
|
||||
func (h *host) statusLocked() *Status {
|
||||
pending := make([]Pending, 0, len(h.pending))
|
||||
for id, offset := range h.pending {
|
||||
pending = append(pending, Pending{ID: id, Offset: offset})
|
||||
}
|
||||
sort.Slice(pending, func(i, j int) bool { return pending[i].Offset < pending[j].Offset })
|
||||
cwd, _ := os.Getwd()
|
||||
status := &Status{
|
||||
Version: Version, SessionID: h.id, HostPID: os.Getpid(),
|
||||
Base: h.base, End: h.end, Acked: h.acked, Busy: h.busy, BusyAtAcked: h.busyAtLocked(h.acked),
|
||||
Init: h.init, Model: h.model, Mode: h.mode, Pending: pending,
|
||||
Exited: h.exited, ExitErr: h.exitErr, Stderr: h.stderr.String(), Directory: cwd,
|
||||
}
|
||||
if h.cmd != nil && h.cmd.Process != nil {
|
||||
status.ClaudePID = h.cmd.Process.Pid
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (h *host) accept() {
|
||||
for {
|
||||
conn, err := h.listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c := &client{conn: conn, gone: make(chan struct{})}
|
||||
go h.serveClient(c)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *host) serveClient(c *client) {
|
||||
defer h.dropClient(c)
|
||||
reader := bufio.NewReaderSize(c.conn, 64<<10)
|
||||
for {
|
||||
line, err := reader.ReadBytes('\n')
|
||||
if len(strings.TrimSpace(string(line))) > 0 {
|
||||
var frame Frame
|
||||
if json.Unmarshal(line, &frame) != nil {
|
||||
_ = c.send(Frame{Op: OpError, Error: "invalid frame"})
|
||||
} else if !h.handle(c, frame) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *host) dropClient(c *client) {
|
||||
c.close()
|
||||
h.mu.Lock()
|
||||
if h.active == c {
|
||||
h.active = nil
|
||||
h.detached = time.Now()
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// handle runs one client frame; false ends the connection.
|
||||
func (h *host) handle(c *client, frame Frame) bool {
|
||||
switch frame.Op {
|
||||
case OpAttach:
|
||||
h.mu.Lock()
|
||||
previous := h.active
|
||||
h.active = c
|
||||
status := h.statusLocked()
|
||||
h.mu.Unlock()
|
||||
if previous != nil && previous != c {
|
||||
previous.close()
|
||||
}
|
||||
if err := c.send(Frame{Op: OpStatus, Status: status}); err != nil {
|
||||
return false
|
||||
}
|
||||
go h.deliver(c, frame.Offset)
|
||||
case OpIn:
|
||||
h.writeIn(frame.Data)
|
||||
case OpAck:
|
||||
h.mu.Lock()
|
||||
if frame.Offset > h.acked && frame.Offset <= h.end {
|
||||
h.acked = frame.Offset
|
||||
h.trimTransitionsLocked()
|
||||
}
|
||||
h.mu.Unlock()
|
||||
case OpStatus:
|
||||
h.mu.Lock()
|
||||
status := h.statusLocked()
|
||||
h.mu.Unlock()
|
||||
_ = c.send(Frame{Op: OpStatus, Status: status})
|
||||
case OpClose:
|
||||
h.stopClaude()
|
||||
_ = c.send(Frame{Op: OpClosed})
|
||||
h.shutdown("closed")
|
||||
return false
|
||||
default:
|
||||
_ = c.send(Frame{Op: OpError, Error: "unknown op " + frame.Op})
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *host) trimTransitionsLocked() {
|
||||
keep := 0
|
||||
for i, t := range h.trans {
|
||||
if t.offset <= h.acked {
|
||||
keep = i
|
||||
}
|
||||
}
|
||||
h.trans = append([]transition(nil), h.trans[keep:]...)
|
||||
}
|
||||
|
||||
func (h *host) writeIn(data json.RawMessage) {
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
var env snoop
|
||||
_ = json.Unmarshal(data, &env)
|
||||
h.mu.Lock()
|
||||
h.activity = time.Now()
|
||||
switch env.Type {
|
||||
case "user":
|
||||
h.setBusyLocked(true, h.end)
|
||||
case "control_response":
|
||||
delete(h.pending, env.Response.RequestID)
|
||||
case "control_request":
|
||||
switch env.Request.Subtype {
|
||||
case "initialize":
|
||||
h.initID = env.RequestID
|
||||
case "set_model":
|
||||
h.modelReqs[env.RequestID] = env.Request.Model
|
||||
case "set_permission_mode":
|
||||
h.modeReqs[env.RequestID] = env.Request.Mode
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
line := append(append([]byte(nil), data...), '\n')
|
||||
h.stdinMu.Lock()
|
||||
defer h.stdinMu.Unlock()
|
||||
if _, err := h.stdin.Write(line); err != nil {
|
||||
log.Printf("write to Claude Code: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// deliver streams spooled lines from offset, then the exit, to one client.
|
||||
func (h *host) deliver(c *client, offset int64) {
|
||||
cursor := offset
|
||||
for {
|
||||
h.mu.Lock()
|
||||
if h.active != c {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
index := sort.Search(len(h.lines), func(i int) bool { return h.lines[i].start >= cursor })
|
||||
if index < len(h.lines) {
|
||||
ref := h.lines[index]
|
||||
h.fileMu.RLock()
|
||||
h.mu.Unlock()
|
||||
data := make([]byte, ref.end-ref.start)
|
||||
_, err := h.spool.ReadAt(data, ref.start-h.base)
|
||||
h.fileMu.RUnlock()
|
||||
if err != nil {
|
||||
log.Printf("spool read: %v", err)
|
||||
c.close()
|
||||
return
|
||||
}
|
||||
if err := c.send(Frame{Op: OpOut, Offset: ref.start, End: ref.end, Data: json.RawMessage(strings.TrimSpace(string(data)))}); err != nil {
|
||||
c.close()
|
||||
return
|
||||
}
|
||||
cursor = ref.end
|
||||
continue
|
||||
}
|
||||
if h.exited && !c.exitSent {
|
||||
c.exitSent = true
|
||||
status := h.statusLocked()
|
||||
h.mu.Unlock()
|
||||
_ = c.send(Frame{Op: OpExit, Status: status, Error: status.ExitErr})
|
||||
// The server has everything; it closes the connection, then we go.
|
||||
go func() {
|
||||
select {
|
||||
case <-c.gone:
|
||||
case <-time.After(30 * time.Second):
|
||||
}
|
||||
h.shutdown("exited")
|
||||
}()
|
||||
continue
|
||||
}
|
||||
changed := h.changed
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case <-changed:
|
||||
case <-c.gone:
|
||||
return
|
||||
case <-h.stopped:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stopClaude ends the CLI: stdin EOF, then an interrupt, then a kill, to the
|
||||
// CLI's whole process group.
|
||||
func (h *host) stopClaude() {
|
||||
if h.cmd == nil || h.cmd.Process == nil {
|
||||
return
|
||||
}
|
||||
h.stdinMu.Lock()
|
||||
_ = h.stdin.Close()
|
||||
h.stdinMu.Unlock()
|
||||
pid := h.cmd.Process.Pid
|
||||
select {
|
||||
case <-h.claudeEnd:
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
_ = syscall.Kill(-pid, syscall.SIGINT)
|
||||
select {
|
||||
case <-h.claudeEnd:
|
||||
return
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
_ = syscall.Kill(-pid, syscall.SIGKILL)
|
||||
select {
|
||||
case <-h.claudeEnd:
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// shutdown stops the CLI, removes the host files and ends Main.
|
||||
func (h *host) shutdown(reason string) {
|
||||
h.mu.Lock()
|
||||
if h.stopping {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.stopping = true
|
||||
active := h.active
|
||||
h.mu.Unlock()
|
||||
log.Printf("stopping (%s)", reason)
|
||||
h.stopClaude()
|
||||
_ = h.listener.Close()
|
||||
Remove(h.dir, h.id)
|
||||
_ = os.Remove(h.paths.Spool + ".next")
|
||||
if active != nil {
|
||||
active.close()
|
||||
}
|
||||
close(h.stopped)
|
||||
}
|
||||
|
||||
// watchIdle releases the CLI while no server is attached: after the idle
|
||||
// expiry when nothing is in flight (the same release the server applies), and
|
||||
// after maxDetached regardless. A CLI that exited with nobody attached is
|
||||
// cleaned up after the idle expiry.
|
||||
func (h *host) watchIdle() {
|
||||
tick := h.idle / 10
|
||||
if tick > 5*time.Second {
|
||||
tick = 5 * time.Second
|
||||
}
|
||||
if tick < 50*time.Millisecond {
|
||||
tick = 50 * time.Millisecond
|
||||
}
|
||||
ticker := time.NewTicker(tick)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-h.stopped:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
now := time.Now()
|
||||
h.mu.Lock()
|
||||
attached := h.active != nil
|
||||
idle := !h.busy && len(h.pending) == 0 && now.Sub(h.detached) > h.idle && now.Sub(h.activity) > h.idle
|
||||
tooLong := now.Sub(h.detached) > h.maxDetached
|
||||
exitedLong := h.exited && now.Sub(h.exitedAt) > h.idle
|
||||
h.mu.Unlock()
|
||||
if attached {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case exitedLong:
|
||||
h.shutdown("exited with no server attached")
|
||||
return
|
||||
case idle:
|
||||
h.shutdown("idle with no server attached")
|
||||
return
|
||||
case tooLong:
|
||||
h.shutdown("no server attached for " + h.maxDetached.String())
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
// Package claudehost keeps one Claude Code CLI process alive across Pipelines
|
||||
// server restarts.
|
||||
//
|
||||
// The server cannot own a `claude -p` process directly and also restart: the
|
||||
// CLI's stdin/stdout pipes die with their parent. A host is a small helper
|
||||
// process in its own session that owns those pipes instead. It spools the
|
||||
// CLI's stream-json output to a bounded per-session file with byte offsets and
|
||||
// serves one local Unix socket (0600, in a 0700 directory) on which the server
|
||||
// writes stdin frames, reads output from an offset, acknowledges what it has
|
||||
// processed, and finally closes the host. A restarted server finds live hosts
|
||||
// through their registry files, reconnects and replays from its last
|
||||
// acknowledged offset.
|
||||
//
|
||||
// The wire format is one JSON object per line in both directions.
|
||||
package claudehost
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Version is the host protocol version reported in every status.
|
||||
const Version = 1
|
||||
|
||||
// Client-to-host operations.
|
||||
const (
|
||||
// OpAttach starts (or restarts) delivery from Frame.Offset. The host
|
||||
// replies with OpStatus, then streams OpOut frames. A newer attach replaces
|
||||
// an older connection.
|
||||
OpAttach = "attach"
|
||||
// OpIn writes Frame.Data, one JSON line, to the CLI's stdin.
|
||||
OpIn = "in"
|
||||
// OpAck commits that output up to Frame.Offset has been processed.
|
||||
OpAck = "ack"
|
||||
// OpStatus asks for a status snapshot without changing delivery.
|
||||
OpStatus = "status"
|
||||
// OpClose stops the CLI and the host. The host replies OpClosed.
|
||||
OpClose = "close"
|
||||
)
|
||||
|
||||
// Host-to-client operations.
|
||||
const (
|
||||
OpOut = "out"
|
||||
OpExit = "exit"
|
||||
OpClosed = "closed"
|
||||
OpError = "error"
|
||||
)
|
||||
|
||||
// Frame is one line on the socket.
|
||||
type Frame struct {
|
||||
Op string `json:"op"`
|
||||
Offset int64 `json:"offset,omitempty"`
|
||||
End int64 `json:"end,omitempty"`
|
||||
Data json.RawMessage `json:"data,omitempty"`
|
||||
Status *Status `json:"status,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Pending is a CLI control request (can_use_tool, a hook) that nobody has
|
||||
// answered yet. It survives a disconnected server.
|
||||
type Pending struct {
|
||||
ID string `json:"id"`
|
||||
Offset int64 `json:"offset"`
|
||||
}
|
||||
|
||||
// Status is the host's snapshot of its CLI.
|
||||
type Status struct {
|
||||
Version int `json:"version"`
|
||||
SessionID string `json:"session_id"`
|
||||
HostPID int `json:"host_pid"`
|
||||
ClaudePID int `json:"claude_pid"`
|
||||
// Base is the oldest offset still in the spool; End is the offset after the
|
||||
// last spooled line; Acked is the last offset a client committed.
|
||||
Base int64 `json:"base"`
|
||||
End int64 `json:"end"`
|
||||
Acked int64 `json:"acked"`
|
||||
// Busy is whether a turn is in flight now; BusyAtAcked is whether one was
|
||||
// in flight at the Acked offset, which is where a reconnecting server's
|
||||
// live processing resumes.
|
||||
Busy bool `json:"busy"`
|
||||
BusyAtAcked bool `json:"busy_at_acked"`
|
||||
// Init is the CLI's initialize response (models, account), so a
|
||||
// reconnecting server does not initialize the CLI twice.
|
||||
Init json.RawMessage `json:"init,omitempty"`
|
||||
// Model and Mode are the CLI's model and permission mode as last applied
|
||||
// (start flags, then successful set_model / set_permission_mode).
|
||||
Model string `json:"model,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Pending []Pending `json:"pending"`
|
||||
Exited bool `json:"exited,omitempty"`
|
||||
ExitErr string `json:"exit_error,omitempty"`
|
||||
Stderr string `json:"stderr,omitempty"`
|
||||
// Directory is the CLI's working directory.
|
||||
Directory string `json:"directory,omitempty"`
|
||||
}
|
||||
|
||||
// Record is the registry file a host writes next to its socket so a
|
||||
// restarted server can find it.
|
||||
type Record struct {
|
||||
Version int `json:"version"`
|
||||
SessionID string `json:"session_id"`
|
||||
HostPID int `json:"host_pid"`
|
||||
ClaudePID int `json:"claude_pid"`
|
||||
Socket string `json:"socket"`
|
||||
Spool string `json:"spool"`
|
||||
Directory string `json:"directory"`
|
||||
Created int64 `json:"created"`
|
||||
}
|
||||
|
||||
// Key is the short file name used for a session's host files. Unix socket
|
||||
// paths are limited to about 100 bytes, so the session id is hashed.
|
||||
func Key(sessionID string) string {
|
||||
sum := sha256.Sum256([]byte(sessionID))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// Paths are one session's files inside a host directory.
|
||||
type Paths struct {
|
||||
Socket, Record, Spool, Log string
|
||||
}
|
||||
|
||||
func PathsFor(dir, sessionID string) Paths {
|
||||
key := Key(sessionID)
|
||||
return Paths{
|
||||
Socket: filepath.Join(dir, key+".sock"),
|
||||
Record: filepath.Join(dir, key+".json"),
|
||||
Spool: filepath.Join(dir, key+".spool"),
|
||||
Log: filepath.Join(dir, key+".log"),
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDir creates a private host directory.
|
||||
func EnsureDir(dir string) error {
|
||||
if strings.TrimSpace(dir) == "" || !filepath.IsAbs(dir) {
|
||||
return errors.New("the Claude host directory must be absolute")
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Chmod(dir, 0o700)
|
||||
}
|
||||
|
||||
// Records lists the registry files in dir. Unreadable files are skipped.
|
||||
func Records(dir string) []Record {
|
||||
matches, _ := filepath.Glob(filepath.Join(dir, "*.json"))
|
||||
records := make([]Record, 0, len(matches))
|
||||
for _, path := range matches {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var record Record
|
||||
if json.Unmarshal(data, &record) != nil || record.SessionID == "" || record.Socket == "" {
|
||||
continue
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
// Remove deletes one session's host files.
|
||||
func Remove(dir, sessionID string) {
|
||||
paths := PathsFor(dir, sessionID)
|
||||
for _, path := range []string{paths.Record, paths.Socket, paths.Spool, paths.Log} {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
|
||||
func writeFileAtomic(path string, data []byte) error {
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
@@ -58,6 +59,13 @@ func handleChatClose(ctx context.Context, request protocol.Message, broker chatB
|
||||
}
|
||||
// A closed chat takes no more turns: prompts it still held are dropped.
|
||||
hub.clearQueue(ctx, sessionID)
|
||||
// A harness that keeps the agent alive across restarts (Claude Code's
|
||||
// host) stops it now; the transcript stays and a later prompt resumes it.
|
||||
if releaser, ok := broker.(chatSessionReleaser); ok {
|
||||
if err := releaser.ReleaseSession(ctx, sessionID); err != nil {
|
||||
log.Printf("chat session %s: release after close: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
session, found, err := registry.OperatorSession(ctx, sessionID)
|
||||
if err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
||||
|
||||
@@ -166,14 +166,32 @@ func (m *chatManager) harness(backend string) chatBroker {
|
||||
return client
|
||||
}
|
||||
|
||||
// chatShutdownDetacher is implemented by a backend whose live sessions can
|
||||
// outlive this server: on shutdown it lets them go instead of stopping them,
|
||||
// and the next server reattaches (Claude Code's detachable hosts).
|
||||
type chatShutdownDetacher interface {
|
||||
Detach()
|
||||
}
|
||||
|
||||
// chatSessionReleaser is implemented by a backend that keeps a session's agent
|
||||
// running until it is explicitly released; chat_close releases it.
|
||||
type chatSessionReleaser interface {
|
||||
ReleaseSession(ctx context.Context, sessionID string) error
|
||||
}
|
||||
|
||||
func closeHarness(client chatBroker) {
|
||||
if detacher, ok := client.(chatShutdownDetacher); ok {
|
||||
detacher.Detach()
|
||||
return
|
||||
}
|
||||
if closer, ok := client.(interface{ Close() }); ok {
|
||||
closer.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// Close releases the owned Codex and Claude Code clients, if any, and stops
|
||||
// later lazy attachment.
|
||||
// Close lets go of the Codex and Claude Code clients, if any, and stops later
|
||||
// lazy attachment. It runs at server shutdown: a backend whose sessions can
|
||||
// survive it (chatShutdownDetacher) is detached, not stopped.
|
||||
func (m *chatManager) Close() {
|
||||
m.mu.Lock()
|
||||
m.closed = true
|
||||
@@ -183,6 +201,19 @@ func (m *chatManager) Close() {
|
||||
closeHarness(claudeClient)
|
||||
}
|
||||
|
||||
// ReleaseSession releases a session's agent on its own harness after an
|
||||
// explicit close. A harness without that notion is left as is.
|
||||
func (m *chatManager) ReleaseSession(ctx context.Context, sessionID string) error {
|
||||
adapter, err := m.brokerFor(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if releaser, ok := adapter.(chatSessionReleaser); ok {
|
||||
return releaser.ReleaseSession(ctx, sessionID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Backends lists the harnesses available for a new session.
|
||||
func (m *chatManager) Backends() []string {
|
||||
backends := []string{}
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ def package(output, version, target):
|
||||
if system not in ("darwin", "linux") or arch not in ("arm64", "amd64"):
|
||||
raise ValueError("Supported: source, darwin-arm64/amd64, linux-arm64/amd64")
|
||||
env = dict(os.environ, GOOS=system, GOARCH=arch, CGO_ENABLED="0")
|
||||
for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines")):
|
||||
for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines"), ("pipelines-claude-host", "pipelines-claude-host")):
|
||||
subprocess.run(["go", "build", "-trimpath", "-o", str(binaries / binary), "./cmd/" + command],
|
||||
cwd=ROOT, env=env, check=True)
|
||||
(binaries / "platform.json").write_text(json.dumps({"os": "Darwin" if system == "darwin" else "Linux", "arch": "arm64" if arch == "arm64" and system == "darwin" else "aarch64" if arch == "arm64" else "x86_64"}))
|
||||
|
||||
@@ -149,6 +149,10 @@ class Setup:
|
||||
content = (f"# {SERVICE_MARKER}\n[Unit]\nDescription=Pipelines native services\n"
|
||||
"\n[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command))
|
||||
+ "\nRestart=on-failure\nRestartSec=5\nTimeoutStopSec=90\nUMask=0077\n"
|
||||
# The supervisor stops its own children. Only it is
|
||||
# signalled, so a Claude Code chat host (its own
|
||||
# session) outlives a restart as it does under launchd.
|
||||
"KillMode=process\n"
|
||||
"\n[Install]\nWantedBy=default.target\n").encode()
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
self._write_atomic(path, content, 0o600)
|
||||
@@ -444,7 +448,9 @@ class Setup:
|
||||
|
||||
# --- build and skill -------------------------------------------------
|
||||
|
||||
COMMANDS = ("pipelines-server", "pipelines")
|
||||
# pipelines-claude-host keeps Claude Code chats alive across server
|
||||
# restarts; the server finds it next to its own executable.
|
||||
COMMANDS = ("pipelines-server", "pipelines", "pipelines-claude-host")
|
||||
|
||||
def build(self, prefix):
|
||||
"""Builds the server and CLI into bin/.next; install_binaries moves
|
||||
|
||||
@@ -305,7 +305,7 @@ class SetupNativeTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "Node.js 18"):
|
||||
installer.configure_web(Path(directory) / "prefix", setup_native.arguments([]))
|
||||
|
||||
def test_build_installs_only_server_and_cli_with_trimpath(self):
|
||||
def test_build_installs_only_server_cli_and_claude_host_with_trimpath(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
recorder = Recorder()
|
||||
installer = self.installer(directory, recorder)
|
||||
@@ -313,11 +313,12 @@ class SetupNativeTests(unittest.TestCase):
|
||||
installer.prepare_layout(prefix)
|
||||
installer.build(prefix)
|
||||
builds = [call for call in recorder.calls if call[:2] == ["go", "build"]]
|
||||
self.assertEqual(len(builds), 2)
|
||||
self.assertEqual(len(builds), 3)
|
||||
for call in builds:
|
||||
self.assertIn("-trimpath", call)
|
||||
self.assertTrue(any(call[-1] == "./cmd/pipelines-server" for call in builds))
|
||||
self.assertTrue(any(call[-1] == "./cmd/pipelines" for call in builds))
|
||||
self.assertTrue(any(call[-1] == "./cmd/pipelines-claude-host" for call in builds))
|
||||
self.assertFalse(any("pipelines-tui" in part for call in recorder.calls for part in call))
|
||||
|
||||
def test_layout_creates_private_prefix_and_copies_manage(self):
|
||||
|
||||
Reference in New Issue
Block a user