diff --git a/Dockerfile b/Dockerfile index 8ff6e3d4..969aea6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ ARG PIPELINES_COMMIT=unknown ARG PIPELINES_DIRTY=unknown RUN CGO_ENABLED=0 go build -trimpath -ldflags "-X git.a-wan.net/alan/pipelines/internal/buildinfo.Version=${PIPELINES_VERSION} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Commit=${PIPELINES_COMMIT} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Dirty=${PIPELINES_DIRTY}" -o /out/pipelines-server ./cmd/pipelines-server RUN CGO_ENABLED=0 go build -trimpath -ldflags "-X git.a-wan.net/alan/pipelines/internal/buildinfo.Version=${PIPELINES_VERSION} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Commit=${PIPELINES_COMMIT} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Dirty=${PIPELINES_DIRTY}" -o /out/pipelines ./cmd/pipelines +RUN CGO_ENABLED=0 go build -trimpath -o /out/pipelines-claude-host ./cmd/pipelines-claude-host FROM ${OPENCODE_IMAGE}@${OPENCODE_DIGEST} ARG OPENCODE_VERSION @@ -42,6 +43,7 @@ RUN apk add --no-cache ca-certificates tzdata git curl nodejs npm python3 \ && chown -R pipelines:pipelines /data /workspace /home/pipelines COPY --from=build /out/pipelines-server /usr/local/bin/pipelines-server COPY --from=build /out/pipelines /usr/local/bin/pipelines +COPY --from=build /out/pipelines-claude-host /usr/local/bin/pipelines-claude-host COPY --from=build /usr/local/go /usr/local/go COPY --chown=10001:10001 . /workspace ENV HOME=/home/pipelines \ diff --git a/cmd/pipelines-claude-host/main.go b/cmd/pipelines-claude-host/main.go new file mode 100644 index 00000000..dec3d79c --- /dev/null +++ b/cmd/pipelines-claude-host/main.go @@ -0,0 +1,14 @@ +// Command pipelines-claude-host keeps one Claude Code CLI process alive across +// Pipelines server restarts. The server starts it; it is not run by hand. See +// internal/claudehost and docs/chat-backends.md. +package main + +import ( + "os" + + "git.a-wan.net/alan/pipelines/internal/claudehost" +) + +func main() { + os.Exit(claudehost.Main(os.Args[1:])) +} diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index 87c5e05e..e2a676a1 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -13,6 +13,7 @@ import ( "time" "git.a-wan.net/alan/pipelines/internal/buildinfo" + "git.a-wan.net/alan/pipelines/internal/claude" "git.a-wan.net/alan/pipelines/internal/discovery" "git.a-wan.net/alan/pipelines/internal/opencode" "git.a-wan.net/alan/pipelines/internal/paths" @@ -148,14 +149,15 @@ func main() { options.OpenCodeStatus = openCodeStatus // One manager routes each chat to its owning harness. OpenCode remains the // default; Codex is attached when its app-server control socket is present. + // Claude Code chats run in detachable hosts scoped to this database, so a + // restart does not end them (internal/claudehost). + claude.ConfigureHosts(claude.DefaultHostDir(*db)) chats := server.NewChatManager(ctx, options.Chat, store) if len(chats.Backends()) > 0 { options.Chat = chats - if detach { - defer chats.DetachSessions() - } else { - defer chats.Close() - } + // Close detaches a backend whose sessions survive a restart + // (chatShutdownDetacher) instead of stopping it. + defer chats.Close() } srv := &http.Server{Addr: *addr, Handler: server.HandlerWithOptions(ctx, options, store), ReadHeaderTimeout: 5 * time.Second} // Deferred before WaitRuns so it runs after the runs settle: the owned stage diff --git a/docs/chat-backends.md b/docs/chat-backends.md index 145b66c8..b6e9b369 100644 --- a/docs/chat-backends.md +++ b/docs/chat-backends.md @@ -25,8 +25,9 @@ the text the harness recorded), and queued prompts are delivered in order. The older start-up probe of recently used chats still runs for anything not recorded, for example after a crash. -Per backend (the contract is `chatSessionDetacher` in -`internal/server/chat_detach.go`): +Per backend (the contract is `chatShutdownDetacher.Detach`, which +`chatManager.Close` calls instead of `Close`, plus `chatSessionReleaser` for +chat_close; see `internal/server/chat_manager.go` and `chat_detach.go`): - **OpenCode**: sessions live in the OpenCode service. An attached endpoint (the operator's CLI-managed service) already outlives the server. Without @@ -39,8 +40,8 @@ Per backend (the contract is `chatSessionDetacher` in control connection, and the next server reconciles through `thread/read`. An approval or question the App Server sent to the old connection is bound to it, so one still pending at shutdown cannot be answered after the restart. -- **Claude Code**: closed as before until its host implements - `DetachSessions`. +- **Claude Code**: its detachable hosts keep the CLI running (see the Claude + Code section). `PIPELINES_SHUTDOWN_SESSIONS=stop` restores the old shutdown, which closes the chat clients and records nothing. See [Installation](install.md#restarts-keep-sessions-running). @@ -82,9 +83,73 @@ credential or alter global settings. `PIPELINES_CLAUDE` may name the executable; otherwise `claude` is resolved from `PATH` and common per-user locations. A new session pins a generated UUID with `--session-id`. When the process is -released (idle or server restart), the next prompt resumes the same conversation -with `--resume `, so context and the selected model survive. Chat creation is -local: the CLI starts on the first prompt or model-catalog read. +released (idle, an explicit close, or a host that crashed), the next prompt +resumes the same conversation with `--resume `, so context and the selected +model survive. Chat creation is local: the CLI starts on the first prompt or +model-catalog read. + +### Surviving a server restart + +A server restart does not stop a Claude Code chat. The installed server runs each +CLI inside `pipelines-claude-host` (`cmd/pipelines-claude-host`, +`internal/claudehost`), a small helper installed next to `pipelines-server`. The +host starts in its own session and process group, so neither launchd nor the +supervisor stops it with the server (the systemd unit uses `KillMode=process` +for the same reason). The host owns the CLI's stdin/stdout. It appends every +output line to a per-session spool file with byte offsets and serves a Unix +socket. The socket is mode 0600 in a 0700 directory: +`/pipelines/claude-hosts//`. Two +servers of one user never adopt each other's chats. Each host also writes a +registry file (`.json`) there. + +The socket speaks one JSON object per line: `attach` (deliver output from an +offset; a newer attach replaces the older connection), `in` (one stdin frame), +`ack` (output processed up to an offset), `status` and `close`. Interrupts, +model and mode switches travel as the CLI's own control requests inside `in`. The +host snoops these frames to keep the CLI's initialize response (models, account), +the applied model and permission mode, a turn's running state, and the control +requests nobody has answered yet (`can_use_tool`, `AskUserQuestion`). + +On shutdown the server only disconnects (`Detach`). On start it reads the +registry and reattaches to every live host before serving. It replays the spool +from its start without broadcasting, which rebuilds the session's stream state, +background commands and agents, and pending approvals and questions. Requests +already answered are skipped. Output after the last acknowledged offset, written +while no server was attached, is then processed live. The turn is shown running +if it was running at that offset, watchers get a resync, and the start-up turn +probe keeps observing it for notifications. An approval or question asked before +the restart can be answered after it. Stop works as usual, and the CLI is never +restarted or initialized twice. + +The host stops the CLI (EOF, then SIGINT, then SIGKILL to its process group) and +removes its files only when one of these happens: + +- The operator closes the chat (`chat_close`). +- The server's idle release or live-process cap fires. +- The CLI exits; the server then gets its stderr. +- With no server attached, the CLI has been idle for the same 20 minutes, or + no server has attached for 24 hours. + +If the host itself dies, the server first tries to reconnect once. If that fails, +it stops the orphaned CLI's process group (only when that pid's command line +still names the session) and removes the host files. A running turn ends with +the notice "the Claude Code host stopped unexpectedly; the next prompt resumes +this session with --resume". The next prompt then resumes the session. + +Limits: + +- A container restart (Docker) still ends every host. So does a reboot or + logout. The transcript survives and `--resume` applies. +- The spool keeps at most 64 MiB of acknowledged output. Older output is + dropped, so background-task state from before that point is not rebuilt after + a restart. Unacknowledged output is dropped only beyond 256 MiB; the server + then logs the gap and clients reload the transcript. +- Turn bookkeeping that lived only in server memory restarts fresh: the turn id, + and which mid-turn messages are still outstanding. Output that begins a + follow-up turn after a reconnect is adopted as a running turn. +- Without the host binary (`PIPELINES_CLAUDE_HOST=off`, or not found next to the + server or on `PATH`), the CLI is the server's child as before and a restart + resumes it. Text and reasoning stream from the CLI's partial-message events; completed messages are de-duplicated against the streamed text. Tool calls appear as normal diff --git a/docs/install.md b/docs/install.md index ca152a6c..adda804b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -42,7 +42,8 @@ Linux needs `systemctl --user`. Other platforms are refused. 2. Builds the Flutter web app in Docker (pinned to `app/.flutter-version`). Builds are cached in `~/.cache/pipelines/app-builds` by a digest of the app sources, so an unchanged app is reused. -3. Builds `pipelines-server` and `pipelines` with `go build -trimpath`. +3. Builds `pipelines-server`, `pipelines` and `pipelines-claude-host` (keeps + Claude Code chats alive across server restarts) with `go build -trimpath`. 4. Installs into the prefix: binaries in `bin/`, web app in `lib/app-web`, web proxy in `lib/web`, `lib/manage.py`, and an empty `lib/app-updates` for APKs. 5. Runs `pipelines skill install` (the authoring skill for OpenCode and Claude @@ -222,11 +223,11 @@ endpoint, keeps running and is reused. Stop it by killing the `pid` in clients as before, add `"PIPELINES_SHUTDOWN_SESSIONS": "stop"` to `opencode_env` in `/pipelines-install.json` and restart. -Limit on Linux: systemd stops every process in the service's cgroup. The -OpenCode services the server starts (stage service, owned chat service) end -with `pipelines stop`/`restart`, and their runs fall back as described above. -An attached OpenCode endpoint and the Codex App Server are separate and -survive. macOS launchd does not kill them. +The OpenCode services the server starts (stage service, owned chat service) +run in their own process session, so macOS launchd leaves them running. The +systemd unit uses `KillMode=process`, so Linux stops only the supervisor, which +stops its own children; the detached services survive. An attached OpenCode +endpoint and the Codex App Server are separate processes anyway. The login service is `~/Library/LaunchAgents/net.a-wan.pipelines.plist` on macOS and `~/.config/systemd/user/net.a-wan.pipelines.service` on Linux. It @@ -238,7 +239,7 @@ autostart, not boot-without-login. | Path | Contents | | --- | --- | | `/pipelines-install.json` | manifest (ports, paths, install args, environment) | -| `/bin/` | `pipelines`, `pipelines-server`, `ntfy` | +| `/bin/` | `pipelines`, `pipelines-server`, `pipelines-claude-host`, `ntfy` | | `/data/pipelines.db` | SQLite database (definitions, runs, chats, settings) | | `/data/artifacts/` | run artifacts and session exports | | `/data/ntfy/` | ntfy config, cache and accounts | diff --git a/internal/claude/chat.go b/internal/claude/chat.go index ff03b1bc..2da75171 100644 --- a/internal/claude/chat.go +++ b/internal/claude/chat.go @@ -325,6 +325,7 @@ func (s *session) ensureProcess(ctx context.Context) (*process, error) { p := s.proc s.lastUsed = time.Now() s.mu.Unlock() + s.awaitReady(ctx) return p, nil } if s.starting != nil { @@ -408,6 +409,7 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error { if s == nil { return nil } + s.awaitReady(ctx) s.mu.Lock() proc := s.proc running := s.running @@ -489,6 +491,7 @@ func (c *Chat) Permissions(ctx context.Context, id string) ([]protocol.ChatPermi if s == nil { return []protocol.ChatPermissionFrame{}, nil } + s.awaitReady(ctx) s.mu.Lock() defer s.mu.Unlock() frames := make([]protocol.ChatPermissionFrame, 0, len(s.approvals)) @@ -505,6 +508,7 @@ func (c *Chat) Forms(ctx context.Context, id string) ([]protocol.ChatForm, error if s == nil { return []protocol.ChatForm{}, nil } + s.awaitReady(ctx) s.mu.Lock() defer s.mu.Unlock() forms := make([]protocol.ChatForm, 0, len(s.forms)) @@ -629,6 +633,7 @@ func (c *Chat) Shells(ctx context.Context, id string) ([]protocol.ChatShell, err if s == nil { return []protocol.ChatShell{}, nil } + s.awaitReady(ctx) s.mu.Lock() defer s.mu.Unlock() result := []protocol.ChatShell{} diff --git a/internal/claude/claude.go b/internal/claude/claude.go index 9d269d5d..817ff701 100644 --- a/internal/claude/claude.go +++ b/internal/claude/claude.go @@ -4,9 +4,11 @@ // mode: stream-json input carries user turns, stream-json output carries // assistant/tool events, and control requests carry interactive prompts // (can_use_tool) and session queries. This adapter owns one CLI process per -// active session and mirrors that protocol into Pipelines chat frames. It never -// changes the operator's Claude configuration, starts a daemon, or invents a -// transcript: history is read back from the CLI's own session files. +// active session and mirrors that protocol into Pipelines chat frames. In the +// installed server each CLI runs inside a per-session pipelines-claude-host so +// it survives a server restart (hosted.go). It never changes the operator's +// Claude configuration or invents a transcript: history is read back from the +// CLI's own session files. package claude import ( @@ -14,6 +16,7 @@ import ( "crypto/rand" "encoding/hex" "errors" + "log" "os" "os/exec" "path/filepath" @@ -22,6 +25,7 @@ import ( "sync" "time" + "git.a-wan.net/alan/pipelines/internal/claudehost" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/workenv" ) @@ -96,6 +100,9 @@ type Chat struct { auth authEnv // dirEnv supplies each chat directory's own environment (direnv, mise). dirEnv *workenv.Resolver + // hostDir is the private directory of detachable CLI hosts, "" when the + // CLI is this server's child (see hosted.go). + hostDir string // hinted records sessions whose registry directory has been resolved. hinted sync.Map @@ -115,13 +122,30 @@ func New(ctx context.Context) *Chat { if ctx == nil { ctx = context.Background() } - ctx, cancel := context.WithCancel(ctx) - c := &Chat{ctx: ctx, cancel: cancel, binary: BinaryPath(), dirEnv: workenv.Default, sessions: map[string]*session{}} + c := newChat(ctx) + if c.hostDir != "" { + if err := claudehost.EnsureDir(c.hostDir); err != nil { + log.Printf("claude chat: host directory %s: %v; Claude Code chats stop on a server restart", c.hostDir, err) + c.hostDir = "" + } + } + c.reattachHosts() go c.reapIdle() return c } -// Close stops every owned CLI process. Session transcripts stay on disk. +// newChat is New without the reattach pass and the reaper (tests). +func newChat(ctx context.Context) *Chat { + ctx, cancel := context.WithCancel(ctx) + c := &Chat{ctx: ctx, cancel: cancel, binary: BinaryPath(), dirEnv: workenv.Default, sessions: map[string]*session{}} + hostConfig.Lock() + c.hostDir = hostConfig.dir + hostConfig.Unlock() + return c +} + +// Close stops every owned CLI process, hosted ones included. Session +// transcripts stay on disk. A server shutdown uses Detach instead. func (c *Chat) Close() { c.cancel() c.mu.Lock() @@ -321,6 +345,10 @@ type session struct { // injectedSeen holds the identities of agent reports and notices already // published live, so a delivery the stream repeats is shown once. injectedSeen map[string]bool + + // silent is set while a reattached host replays output an earlier server + // already published: state is rebuilt, nothing is broadcast. + silent bool } type promptRecord struct { @@ -355,6 +383,10 @@ func (s *session) emit(frame protocol.ChatEventFrame) { s.compaction = nil } } + if s.silent { + s.mu.Unlock() + return + } targets := make([]chan protocol.ChatEventFrame, 0, len(s.subs)) for ch := range s.subs { targets = append(targets, ch) diff --git a/internal/claude/hosted.go b/internal/claude/hosted.go new file mode 100644 index 00000000..dc42d4b7 --- /dev/null +++ b/internal/claude/hosted.go @@ -0,0 +1,591 @@ +package claude + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// Detachable hosts. With hosting configured, a session's CLI is owned by a +// pipelines-claude-host process (internal/claudehost) instead of this server, +// so a server restart does not end the conversation: the next server finds +// the host's registry file, reconnects, rebuilds the session's state from the +// spooled output and continues from its last acknowledged offset. Without +// hosting (tests, a missing host binary) the CLI is this server's child, as +// before, and a restart resumes it with --resume. + +const hostAttachTimeout = 3 * time.Second + +var hostConfig struct { + sync.Mutex + dir string + binary string + args []string +} + +// ConfigureHosts makes Chats created afterwards run their CLIs in detachable +// hosts kept in dir. An empty dir turns hosting off. PIPELINES_CLAUDE_HOST +// names the host executable ("off" disables hosting); otherwise +// pipelines-claude-host is looked up next to this executable, then on PATH. +func ConfigureHosts(dir string) { + hostConfig.Lock() + defer hostConfig.Unlock() + hostConfig.dir = strings.TrimSpace(dir) + hostConfig.binary = "" + hostConfig.args = nil + if hostConfig.dir == "" { + return + } + binary := hostBinaryPath() + if binary == "" { + log.Printf("claude chat: pipelines-claude-host was not found; Claude Code chats stop on a server restart") + hostConfig.dir = "" + return + } + hostConfig.binary = binary +} + +// DefaultHostDir is the private host directory for one server instance, +// scoped by its database path so two servers of one user never adopt each +// other's chats. +func DefaultHostDir(scope string) string { + base, err := os.UserCacheDir() + if err != nil || base == "" { + return "" + } + if absolute, err := filepath.Abs(scope); err == nil { + scope = absolute + } + sum := sha256.Sum256([]byte(scope)) + return filepath.Join(base, "pipelines", "claude-hosts", hex.EncodeToString(sum[:6])) +} + +func hostBinaryPath() string { + if value := strings.TrimSpace(os.Getenv("PIPELINES_CLAUDE_HOST")); value != "" { + if value == "off" { + return "" + } + return value + } + if executable, err := os.Executable(); err == nil { + candidate := filepath.Join(filepath.Dir(executable), "pipelines-claude-host") + if info, err := os.Stat(candidate); err == nil && !info.IsDir() { + return candidate + } + } + if path, err := exec.LookPath("pipelines-claude-host"); err == nil { + return path + } + return "" +} + +// hostLink is a process's connection to its host. +type hostLink struct { + dir string + record claudehost.Record + + mu sync.Mutex + conn *claudehost.Conn + committed int64 + + detached atomic.Bool + closing atomic.Bool +} + +func (l *hostLink) current() *claudehost.Conn { + l.mu.Lock() + defer l.mu.Unlock() + return l.conn +} + +func (l *hostLink) send(frame claudehost.Frame) error { + conn := l.current() + if conn == nil { + return errProcessClosed + } + if err := conn.Send(frame); err != nil { + return fmt.Errorf("write to the Claude Code host: %w", err) + } + return nil +} + +// replayPlan says how a freshly attached reader treats output it has seen +// before: lines below silentUntil were processed by an earlier server and only +// rebuild state; control requests below catchUpEnd are applied only when the +// host still has them pending (answered ones must not reappear). +type replayPlan struct { + base int64 + silentUntil int64 + catchUpEnd int64 + pending map[string]bool + busyAtAcked bool + restore bool +} + +// startHosted starts a host for args and attaches to it. +func (s *session) startHosted(ctx context.Context, args []string, mode string, env []string) (*process, error) { + c := s.chat + hostConfig.Lock() + binary, hostArgs := hostConfig.binary, append([]string(nil), hostConfig.args...) + hostConfig.Unlock() + cmd, err := claudehost.Spawn(ctx, claudehost.SpawnOptions{ + HostBinary: binary, HostArgs: hostArgs, Dir: c.hostDir, SessionID: s.id, + Claude: c.binary, Args: args, Env: env, Cwd: s.directory, Idle: idleProcessExpiry, + }) + if err != nil { + return nil, err + } + // Reap the host if it ends while this server is still its parent. + go func() { _ = cmd.Wait() }() + paths := claudehost.PathsFor(c.hostDir, s.id) + conn, status, err := claudehost.Attach(paths.Socket, 0, hostAttachTimeout) + if err != nil { + claudehost.KillStale(c.hostDir, claudehost.Record{SessionID: s.id, HostPID: cmd.Process.Pid}) + return nil, fmt.Errorf("attach to the Claude Code host: %w", err) + } + link := &hostLink{dir: c.hostDir, conn: conn, record: claudehost.Record{ + SessionID: s.id, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: paths.Socket, + }} + p := newProcess(s, mode) + p.link = link + p.markReady() + go p.readHost(conn, replayPlan{}) + return p, nil +} + +// connectHost attaches to a host found in the registry and prepares a +// process for s without registering it. A host that is unreachable, belongs +// to another session or never finished initializing is stopped and its files +// removed; nil is returned and the next prompt starts a fresh CLI with +// --resume. +func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) { + conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout) + if err != nil { + log.Printf("claude chat %s: host unreachable (%v); the next prompt resumes the session", record.SessionID, err) + claudehost.KillStale(c.hostDir, record) + return nil, nil, replayPlan{} + } + var models []protocol.ChatModel + var account accountIdentity + if len(status.Init) > 0 { + models, account, err = parseInitialize(status.Init) + } else { + err = errors.New("the host's CLI never finished initializing") + } + if status.SessionID != record.SessionID || err != nil { + _ = conn.Close() + claudehost.KillStale(c.hostDir, record) + return nil, nil, replayPlan{} + } + directory := status.Directory + if directory == "" { + directory = record.Directory + } + s := c.sessionFor(record.SessionID, directory) + p := newProcess(s, status.Mode) + p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, record: claudehost.Record{ + SessionID: record.SessionID, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: record.Socket, + }} + p.stderr.WriteString(status.Stderr) + p.models, p.account, p.appliedModel = models, account, status.Model + plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true} + for _, pending := range status.Pending { + plan.pending[pending.ID] = true + } + if status.Acked < status.Base { + // Output the previous server never processed was dropped by the spool + // bound; watchers reload the transcript. + log.Printf("claude chat %s: %d bytes of unread output exceeded the host spool", record.SessionID, status.Base-status.Acked) + plan.silentUntil = status.Base + } + return p, conn, plan +} + +// adoptLocked applies an attached host's model and account to the session. +// Callers hold s.mu. +func (s *session) adoptLocked(p *process) { + s.models, s.account = p.models, p.account + s.appliedModel = p.appliedModel + if s.model == "" { + s.model = p.appliedModel + } + s.lastUsed = time.Now() +} + +// attachHost adopts a live host found in the registry at server start. +func (c *Chat) attachHost(record claudehost.Record) bool { + p, conn, plan := c.connectHost(record) + if p == nil { + return false + } + s := p.session + s.mu.Lock() + if s.proc != nil || s.starting != nil { + s.mu.Unlock() + _ = conn.Close() + return false + } + s.proc = p + s.adoptLocked(p) + s.mu.Unlock() + go p.readHost(conn, plan) + return true +} + +// reattachHosts adopts every live host of this server's directory. The +// registration is synchronous, so a prompt never races a second host for the +// same session; the replay that rebuilds each session runs in the background +// (see process.ready). +func (c *Chat) reattachHosts() { + if c.hostDir == "" { + return + } + var wg sync.WaitGroup + for _, record := range claudehost.Records(c.hostDir) { + wg.Add(1) + go func(record claudehost.Record) { + defer wg.Done() + if c.attachHost(record) { + log.Printf("claude chat %s: reattached to its running Claude Code host", record.SessionID) + } + }(record) + } + wg.Wait() +} + +// existingHost adopts a host left for this session (one that started after +// this server's reattach pass) instead of starting a second one. The caller +// holds the session's starting slot and registers the process. +func (s *session) existingHost() *process { + c := s.chat + if c.hostDir == "" { + return nil + } + data, err := os.ReadFile(claudehost.PathsFor(c.hostDir, s.id).Record) + if err != nil { + return nil + } + var record claudehost.Record + if json.Unmarshal(data, &record) != nil || record.SessionID != s.id { + claudehost.Remove(c.hostDir, s.id) + return nil + } + p, conn, plan := c.connectHost(record) + if p == nil { + return nil + } + s.mu.Lock() + s.adoptLocked(p) + s.mu.Unlock() + go p.readHost(conn, plan) + return p +} + +// readHost is the hosted counterpart of readStdout + wait. +func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) { + s := p.session + live := false + goLive := func() { + if live { + return + } + live = true + s.mu.Lock() + s.silent = false + turn := "" + if plan.restore { + if plan.busyAtAcked && !s.running { + turn = fmt.Sprintf("claude-%d", time.Now().UnixNano()) + s.turnID = turn + s.running = true + s.steered = 0 + } else if !plan.busyAtAcked { + // Output that starts a turn after the reconnect (a follow-up of + // messages written before it) is adopted as a running turn. + s.awaitFollowUp = true + } + } + s.mu.Unlock() + if turn != "" { + s.emit(protocol.ChatEventFrame{SessionID: s.id, NativeTurnID: turn, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: turn}}) + } + if plan.restore { + // Watchers already attached reload the transcript. + s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventResync}) + } + } + if plan.silentUntil > plan.base { + s.mu.Lock() + s.silent = true + s.mu.Unlock() + } else { + goLive() + } + if plan.catchUpEnd <= plan.base { + p.markReady() + } + sinceAck := 0 + for { + frame, err := conn.Recv() + if err != nil { + goLive() + p.markReady() + if next := p.hostLost(conn, err); next != nil { + conn = next + plan = replayPlan{} + continue + } + return + } + switch frame.Op { + case claudehost.OpOut: + if frame.Offset >= plan.silentUntil { + goLive() + } + if !(frame.Offset < plan.catchUpEnd && answeredRequest(frame.Data, plan.pending)) { + s.handleLine(p, frame.Data) + } + p.link.mu.Lock() + p.link.committed = frame.End + p.link.mu.Unlock() + sinceAck++ + if sinceAck >= 64 || !conn.Buffered() { + _ = conn.Send(claudehost.Frame{Op: claudehost.OpAck, Offset: frame.End}) + sinceAck = 0 + } + if frame.End >= plan.silentUntil { + goLive() + } + if frame.End >= plan.catchUpEnd { + p.markReady() + } + case claudehost.OpExit: + goLive() + p.markReady() + if frame.Status != nil && frame.Status.Stderr != "" { + p.mu.Lock() + if p.stderr.Len() == 0 { + p.stderr.WriteString(frame.Status.Stderr) + } + p.mu.Unlock() + } + var exitErr error + if frame.Error != "" { + exitErr = errors.New(frame.Error) + } + p.link.closing.Store(true) + _ = conn.Close() + p.finish(exitErr) + return + case claudehost.OpClosed: + p.link.closing.Store(true) + } + } +} + +// answeredRequest reports a replayed CLI control request that is no longer +// pending, so an approval already answered is not shown again. +func answeredRequest(line []byte, pending map[string]bool) bool { + var envelope struct { + Type string `json:"type"` + RequestID string `json:"request_id"` + } + if json.Unmarshal(line, &envelope) != nil || envelope.Type != "control_request" { + return false + } + return !pending[envelope.RequestID] +} + +func (p *process) markReady() { + p.readyOnce.Do(func() { close(p.ready) }) +} + +// hostLost handles an ended host connection. A deliberate detach or close just +// ends the reader; an unexpected loss is retried once (the host may only have +// dropped the connection), and otherwise the host is treated as crashed: what +// it left is stopped and the turn is settled with a notice, and the next prompt +// resumes the session with --resume as after any release. +func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn { + link := p.link + _ = conn.Close() + if link.detached.Load() { + p.finishDetached() + return nil + } + if link.closing.Load() { + p.finish(nil) + return nil + } + link.mu.Lock() + committed := link.committed + link.mu.Unlock() + for attempt := 0; attempt < 2; attempt++ { + time.Sleep(150 * time.Millisecond) + if link.detached.Load() || link.closing.Load() { + break + } + next, _, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout) + if err == nil { + link.mu.Lock() + link.conn = next + link.mu.Unlock() + return next + } + } + if link.detached.Load() { + p.finishDetached() + return nil + } + if link.closing.Load() { + p.finish(nil) + return nil + } + log.Printf("claude chat %s: host connection lost (%v); falling back to --resume", p.session.id, cause) + claudehost.KillStale(link.dir, link.record) + p.mu.Lock() + p.stderr.Reset() + p.mu.Unlock() + p.finish(errors.New("the Claude Code host stopped unexpectedly; the next prompt resumes this session with --resume")) + return nil +} + +// finishDetached ends a process this server let go of on shutdown. The CLI +// keeps running in its host; nothing is reported to watchers. +func (p *process) finishDetached() { + p.mu.Lock() + if p.exited { + p.mu.Unlock() + return + } + p.exited = true + pending := p.pending + p.pending = map[string]chan controlReply{} + close(p.done) + p.mu.Unlock() + for _, ch := range pending { + select { + case ch <- controlReply{err: "the server is shutting down"}: + default: + } + } +} + +// closeHosted stops the host and its CLI (an explicit release). +func (p *process) closeHosted() { + link := p.link + link.closing.Store(true) + if err := link.send(claudehost.Frame{Op: claudehost.OpClose}); err != nil { + claudehost.KillStale(link.dir, link.record) + } + select { + case <-p.done: + case <-time.After(8 * time.Second): + claudehost.KillStale(link.dir, link.record) + if conn := link.current(); conn != nil { + _ = conn.Close() + } + } +} + +// detach lets the host keep running while this server stops. +func (p *process) detach() { + link := p.link + if link == nil { + p.close() + return + } + link.detached.Store(true) + if conn := link.current(); conn != nil { + _ = conn.Close() + } + select { + case <-p.done: + case <-time.After(2 * time.Second): + } +} + +// Detach disconnects from every hosted CLI without stopping it, for a server +// shutdown: the next server reattaches. CLIs this server owns directly (no +// host) are stopped as Close does. +func (c *Chat) Detach() { + c.cancel() + c.mu.Lock() + c.closed = true + sessions := make([]*session, 0, len(c.sessions)) + for _, s := range c.sessions { + sessions = append(sessions, s) + } + c.sessions = map[string]*session{} + c.mu.Unlock() + var wg sync.WaitGroup + for _, s := range sessions { + s.mu.Lock() + p := s.proc + s.proc = nil + s.mu.Unlock() + if p == nil { + continue + } + wg.Add(1) + go func() { + defer wg.Done() + p.detach() + }() + } + wg.Wait() +} + +// ReleaseSession stops a session's CLI (and its host) for an explicit operator +// close. The transcript stays; a later prompt resumes it. +func (c *Chat) ReleaseSession(_ context.Context, id string) error { + if s := c.session(id); s != nil { + s.closeProcess() + } + return nil +} + +// ProbeTurn reports whether a turn is in flight, after a reattached host's +// replay has rebuilt the session, so a turn that survived a restart is +// observed and notifies. +func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) { + s := c.session(id) + if s == nil { + return false, nil + } + s.awaitReady(ctx) + s.mu.Lock() + defer s.mu.Unlock() + return s.running, nil +} + +// awaitReady waits, bounded, until a reattached process has replayed what it +// missed, so approvals and the running turn are known before they are read. +func (s *session) awaitReady(ctx context.Context) { + s.mu.Lock() + p := s.proc + s.mu.Unlock() + if p == nil { + return + } + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + select { + case <-p.ready: + case <-ctx.Done(): + case <-timer.C: + } +} diff --git a/internal/claude/hosted_test.go b/internal/claude/hosted_test.go new file mode 100644 index 00000000..a861e6cb --- /dev/null +++ b/internal/claude/hosted_test.go @@ -0,0 +1,385 @@ +package claude + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/claudehost" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// The test binary doubles as pipelines-claude-host: a host spawned by a test +// re-executes it with this variable set. +const hostSelfTest = "PIPELINES_CLAUDE_HOST_SELFTEST" + +func TestMain(m *testing.M) { + if os.Getenv(hostSelfTest) == "1" { + os.Exit(claudehost.Main(os.Args[1:])) + } + os.Exit(m.Run()) +} + +type hostedFixture struct { + hostDir string + gate string + logPath string +} + +// hostedSetup configures the fake CLI and detachable hosts in a short private +// directory (Unix socket paths are length-limited). +func hostedSetup(t *testing.T) hostedFixture { + t.Helper() + base, err := os.MkdirTemp("/tmp", "pch") + if err != nil { + t.Fatal(err) + } + source, err := os.ReadFile(filepath.Join("testdata", "fake-claude.py")) + if err != nil { + t.Fatal(err) + } + script := filepath.Join(base, "claude") + if err := os.WriteFile(script, source, 0o700); err != nil { + t.Fatal(err) + } + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + fixture := hostedFixture{hostDir: filepath.Join(base, "h"), gate: filepath.Join(base, "gate"), logPath: filepath.Join(base, "args.log")} + t.Setenv("PIPELINES_CLAUDE", script) + t.Setenv("CLAUDE_CONFIG_DIR", filepath.Join(base, "config")) + t.Setenv("FAKE_CLAUDE_LOG", fixture.logPath) + t.Setenv("FAKE_CLAUDE_GATE", fixture.gate) + t.Setenv("PIPELINES_CLAUDE_HOST", executable) + t.Setenv(hostSelfTest, "1") + ConfigureHosts(fixture.hostDir) + t.Cleanup(func() { + ConfigureHosts("") + for _, record := range claudehost.Records(fixture.hostDir) { + claudehost.KillStale(fixture.hostDir, record) + } + _ = os.RemoveAll(base) + }) + return fixture +} + +func hostedChat(t *testing.T) *Chat { + t.Helper() + chat := New(context.Background()) + if chat.hostDir == "" { + t.Fatal("hosting is not configured") + } + t.Cleanup(chat.Close) + return chat +} + +func onlyRecord(t *testing.T, fixture hostedFixture) claudehost.Record { + t.Helper() + var records []claudehost.Record + eventually(t, "a host registry record", func() bool { + records = claudehost.Records(fixture.hostDir) + return len(records) == 1 + }) + return records[0] +} + +func alive(pid int) bool { return pid > 0 && syscall.Kill(pid, 0) == nil } + +func textFrame(contains string) func(protocol.ChatEventFrame) bool { + return func(frame protocol.ChatEventFrame) bool { + return strings.Contains(frame.Delta, contains) || (frame.Part != nil && strings.Contains(frame.Part.Text, contains)) + } +} + +func stepStarted(frame protocol.ChatEventFrame) bool { + return frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepStarted +} + +// restart stands in for a server restart: the old Chat detaches (its host +// keeps running) and a new one is built over the same host directory. +func restart(t *testing.T, old *Chat) *Chat { + t.Helper() + old.Detach() + return hostedChat(t) +} + +func TestHostedTurnSurvivesRestartAndReplaysMissedOutput(t *testing.T) { + fixture := hostedSetup(t) + first := hostedChat(t) + session := newCreatedChat(t, first) + frames := subscribe(t, first, session.ID) + if err := first.Prompt(context.Background(), session.ID, "gate please"); err != nil { + t.Fatal(err) + } + waitFrame(t, frames, "text before the restart", textFrame("Working")) + record := onlyRecord(t, fixture) + first.Detach() + if !alive(record.HostPID) || !alive(record.ClaudePID) { + t.Fatal("detaching must leave the host and its CLI running") + } + + // The CLI finishes its turn while no server is attached. + if err := os.WriteFile(fixture.gate, nil, 0o600); err != nil { + t.Fatal(err) + } + eventually(t, "the turn result in the spool", func() bool { + data, _ := os.ReadFile(record.Spool) + return strings.Contains(string(data), `"type":"result"`) + }) + + second := newChat(context.Background()) + t.Cleanup(second.Close) + second.sessionFor(session.ID, session.Directory) + frames = subscribe(t, second, session.ID) + second.reattachHosts() + waitFrame(t, frames, "the restored running turn", stepStarted) + waitFrame(t, frames, "text produced while disconnected", textFrame("after the restart")) + waitStepEnded(t, frames) + if second.TurnRunning(session.ID) { + t.Fatal("the replayed result must settle the turn") + } + if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 { + t.Fatalf("the CLI must not be restarted, spawns: %+v", spawns) + } + // The same CLI keeps serving turns. + if err := second.Prompt(context.Background(), session.ID, "hello again"); err != nil { + t.Fatal(err) + } + waitFrame(t, frames, "the next turn's text", textFrame("Hello")) + waitStepEnded(t, frames) +} + +func TestPendingApprovalIsAnsweredAfterRestart(t *testing.T) { + hostedSetup(t) + first := hostedChat(t) + session := newCreatedChat(t, first) + if err := first.Prompt(context.Background(), session.ID, "permission please"); err != nil { + t.Fatal(err) + } + var before []protocol.ChatPermissionFrame + eventually(t, "the approval before the restart", func() bool { + before, _ = first.Permissions(context.Background(), session.ID) + return len(before) == 1 + }) + second := restart(t, first) + running, err := second.ProbeTurn(context.Background(), session.ID) + if err != nil || !running { + t.Fatalf("the restored turn must be running: %v %v", running, err) + } + after, err := second.Permissions(context.Background(), session.ID) + if err != nil || len(after) != 1 || after[0].RequestID != before[0].RequestID { + t.Fatalf("the pending approval must survive the restart: %+v %v", after, err) + } + frames := subscribe(t, second, session.ID) + if err := second.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: after[0].RequestID, Decision: protocol.ChatDecisionOnce}); err != nil { + t.Fatal(err) + } + waitFrame(t, frames, "the approved tool result", func(frame protocol.ChatEventFrame) bool { + return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil && + frame.Part.State.Status == protocol.ChatToolCompleted && len(frame.Part.State.Content) == 1 && + strings.Contains(frame.Part.State.Content[0].Text, "File created") + }) + waitStepEnded(t, frames) + + // An answered approval is not shown again by a later restart's replay. + third := restart(t, second) + if pending, _ := third.Permissions(context.Background(), session.ID); len(pending) != 0 { + t.Fatalf("an answered approval reappeared: %+v", pending) + } + if third.TurnRunning(session.ID) { + t.Fatal("a finished turn must not be restored as running") + } +} + +func TestQuestionSurvivesRestart(t *testing.T) { + hostedSetup(t) + first := hostedChat(t) + session := newCreatedChat(t, first) + if err := first.Prompt(context.Background(), session.ID, "question please"); err != nil { + t.Fatal(err) + } + eventually(t, "the question before the restart", func() bool { + forms, _ := first.Forms(context.Background(), session.ID) + return len(forms) == 1 + }) + second := restart(t, first) + forms, err := second.Forms(context.Background(), session.ID) + if err != nil || len(forms) != 1 { + t.Fatalf("the question must survive the restart: %+v %v", forms, err) + } + frames := subscribe(t, second, session.ID) + if err := second.Answer(context.Background(), protocol.ChatAnswerRequest{SessionID: session.ID, FormID: forms[0].ID, Answers: map[string]any{"q0": "Beta"}}); err != nil { + t.Fatal(err) + } + waitFrame(t, frames, "the answered question", func(frame protocol.ChatEventFrame) bool { + return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil && + len(frame.Part.State.Content) == 1 && strings.Contains(frame.Part.State.Content[0].Text, "Beta") + }) + waitStepEnded(t, frames) +} + +func TestInterruptAfterRestart(t *testing.T) { + hostedSetup(t) + first := hostedChat(t) + session := newCreatedChat(t, first) + frames := subscribe(t, first, session.ID) + if err := first.Prompt(context.Background(), session.ID, "long task"); err != nil { + t.Fatal(err) + } + waitFrame(t, frames, "the long turn's text", textFrame("Hello")) + second := restart(t, first) + if running, _ := second.ProbeTurn(context.Background(), session.ID); !running { + t.Fatal("the long turn must still be running after the restart") + } + frames = subscribe(t, second, session.ID) + if err := second.Interrupt(context.Background(), session.ID); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + if second.TurnRunning(session.ID) { + t.Fatal("the interrupted turn must settle") + } +} + +func TestModelAndBackgroundTaskSurviveRestart(t *testing.T) { + fixture := hostedSetup(t) + first := hostedChat(t) + session := newCreatedChat(t, first) + if _, err := first.Models(context.Background(), session.ID); err != nil { + t.Fatal(err) + } + if err := first.SwitchModel(context.Background(), session.ID, Backend, "haiku"); err != nil { + t.Fatal(err) + } + frames := subscribe(t, first, session.ID) + if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + if shells, _ := first.Shells(context.Background(), session.ID); len(shells) != 1 { + t.Fatalf("the background command must be running: %+v", shells) + } + second := restart(t, first) + shells, err := second.Shells(context.Background(), session.ID) + if err != nil || len(shells) != 1 || shells[0].Command != "sleep 600" { + t.Fatalf("the background command must still be listed after the restart: %+v %v", shells, err) + } + s := second.session(session.ID) + s.mu.Lock() + applied, models := s.appliedModel, len(s.models) + s.mu.Unlock() + if applied != "haiku" || models != 3 { + t.Fatalf("the live model and catalog must be restored: %q %d", applied, models) + } + frames = subscribe(t, second, session.ID) + if err := second.PromptWithMessageIDModel(context.Background(), session.ID, "notify me", "", "haiku"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + if shells, _ := second.Shells(context.Background(), session.ID); len(shells) != 0 { + t.Fatalf("the notification must finish the restored command: %+v", shells) + } + if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 { + t.Fatalf("the CLI must not be restarted, spawns: %d", len(spawns)) + } +} + +func TestHostCrashFallsBackToResume(t *testing.T) { + fixture := hostedSetup(t) + chat := hostedChat(t) + session := newCreatedChat(t, chat) + frames := subscribe(t, chat, session.ID) + if err := chat.Prompt(context.Background(), session.ID, "long task"); err != nil { + t.Fatal(err) + } + waitFrame(t, frames, "the long turn's text", textFrame("Hello")) + record := onlyRecord(t, fixture) + if err := syscall.Kill(record.HostPID, syscall.SIGKILL); err != nil { + t.Fatal(err) + } + waitFrame(t, frames, "the host crash notice", func(frame protocol.ChatEventFrame) bool { + return frame.Part != nil && frame.Part.State != nil && frame.Part.State.Error != nil && + strings.Contains(frame.Part.State.Error.Message, "--resume") + }) + waitStepEnded(t, frames) + eventually(t, "the orphaned CLI to stop", func() bool { return !alive(record.ClaudePID) || isZombie(record.ClaudePID) }) + if _, err := os.Stat(record.Socket); !os.IsNotExist(err) { + t.Fatalf("the crashed host's files must be removed: %v", err) + } + if err := chat.Prompt(context.Background(), session.ID, "hello again"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + spawns := readArgs(t, fixture.logPath) + if len(spawns) != 2 || argValue(spawns[1].Args, "--resume") != session.ID { + t.Fatalf("the next prompt must resume the session: %+v", spawns) + } +} + +func isZombie(pid int) bool { + var status syscall.WaitStatus + got, _ := syscall.Wait4(pid, &status, syscall.WNOHANG, nil) + return got == pid +} + +func TestReleasingAHostedChatStopsItsHost(t *testing.T) { + fixture := hostedSetup(t) + chat := hostedChat(t) + session := newCreatedChat(t, chat) + frames := subscribe(t, chat, session.ID) + if err := chat.Prompt(context.Background(), session.ID, "hello"); err != nil { + t.Fatal(err) + } + waitStepEnded(t, frames) + record := onlyRecord(t, fixture) + if err := chat.ReleaseSession(context.Background(), session.ID); err != nil { + t.Fatal(err) + } + eventually(t, "the host to exit", func() bool { return !alive(record.HostPID) }) + eventually(t, "the CLI to exit", func() bool { return !alive(record.ClaudePID) || isZombie(record.ClaudePID) }) + if records := claudehost.Records(fixture.hostDir); len(records) != 0 { + t.Fatalf("a released host must remove its registry record: %+v", records) + } + // A restarted server finds nothing to reattach; the next prompt resumes. + next := hostedChat(t) + if next.session(session.ID) != nil { + t.Fatal("a released session must not be reattached") + } +} + +func TestHostStatusReportsRegistryAndPrivateSocket(t *testing.T) { + fixture := hostedSetup(t) + chat := hostedChat(t) + session := newCreatedChat(t, chat) + if _, err := chat.Models(context.Background(), session.ID); err != nil { + t.Fatal(err) + } + record := onlyRecord(t, fixture) + info, err := os.Stat(record.Socket) + if err != nil || info.Mode().Perm()&0o077 != 0 { + t.Fatalf("the socket must be private: %v %v", info.Mode(), err) + } + dir, err := os.Stat(fixture.hostDir) + if err != nil || dir.Mode().Perm() != 0o700 { + t.Fatalf("the host directory must be private: %v %v", dir.Mode(), err) + } + conn, err := claudehost.Dial(record.Socket, time.Second) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := conn.Send(claudehost.Frame{Op: claudehost.OpStatus}); err != nil { + t.Fatal(err) + } + frame, err := conn.Recv() + if err != nil || frame.Status == nil || frame.Status.SessionID != session.ID || len(frame.Status.Init) == 0 { + raw, _ := json.Marshal(frame) + t.Fatalf("status: %s %v", raw, err) + } +} diff --git a/internal/claude/process.go b/internal/claude/process.go index 898abd87..08318997 100644 --- a/internal/claude/process.go +++ b/internal/claude/process.go @@ -14,6 +14,7 @@ import ( "sync/atomic" "time" + "git.a-wan.net/alan/pipelines/internal/claudehost" "git.a-wan.net/alan/pipelines/internal/protocol" ) @@ -48,6 +49,23 @@ type process struct { // mode is the permission mode this process runs in, guarded by the // session's mu: set at spawn and after a confirmed set_permission_mode. mode string + + // link is set when a detachable host owns the CLI (hosted.go); cmd and + // stdin are then unused. + link *hostLink + // ready closes once a reattached process has replayed the output it + // missed; a process this server started is ready at once. + ready chan struct{} + readyOnce sync.Once + // models, account and appliedModel carry an attached host's + // initialize state until the session adopts it. + models []protocol.ChatModel + account accountIdentity + appliedModel string +} + +func newProcess(s *session, mode string) *process { + return &process{session: s, pending: map[string]chan controlReply{}, done: make(chan struct{}), stderrDone: make(chan struct{}), ready: make(chan struct{}), mode: mode} } type controlReply struct { @@ -97,6 +115,7 @@ func sessionDisplayName(title string) string { // spawn starts the CLI in the session directory. A session whose transcript // already exists resumes it; a new one pins the id we recorded at create time. +// With hosting configured the CLI runs in a detachable host (hosted.go). func (s *session) spawn(ctx context.Context) (*process, error) { if err := s.requireBinary(); err != nil { return nil, err @@ -107,34 +126,24 @@ func (s *session) spawn(ctx context.Context) (*process, error) { if info, err := os.Stat(s.directory); err != nil || !info.IsDir() { return nil, fmt.Errorf("Claude session directory is not available: %s", s.directory) } + if p := s.existingHost(); p != nil { + return p, nil + } args, mode := s.cliArgs(findTranscriptFile(s.id) != "") - cmd := exec.CommandContext(s.chat.ctx, s.chat.binary, args...) - cmd.Dir = s.directory env, notices := s.chat.processEnv(ctx, s.directory) - cmd.Env = env s.mu.Lock() s.envNotices = notices s.mu.Unlock() - cmd.WaitDelay = 5 * time.Second - stdin, err := cmd.StdinPipe() + var p *process + var err error + if s.chat.hostDir != "" { + p, err = s.startHosted(ctx, args, mode, env) + } else { + p, err = s.startDirect(args, mode, env) + } if err != nil { return nil, err } - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("start Claude Code: %w", err) - } - p := &process{session: s, cmd: cmd, stdin: stdin, pending: map[string]chan controlReply{}, done: make(chan struct{}), stderrDone: make(chan struct{}), mode: mode} - go p.readStderr(stderr) - go p.readStdout(stdout) - go p.wait() // The CLI holds can_use_tool requests until the host has initialized. Do that // before any user turn so an approval is never silently denied. initCtx, cancel := context.WithTimeout(ctx, initializeTimeout) @@ -152,11 +161,47 @@ func (s *session) spawn(ctx context.Context) (*process, error) { return p, nil } +// startDirect runs the CLI as this server's own child. +func (s *session) startDirect(args []string, mode string, env []string) (*process, error) { + cmd := exec.CommandContext(s.chat.ctx, s.chat.binary, args...) + cmd.Dir = s.directory + cmd.Env = env + cmd.WaitDelay = 5 * time.Second + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start Claude Code: %w", err) + } + p := newProcess(s, mode) + p.cmd, p.stdin = cmd, stdin + p.markReady() + go p.readStderr(stderr) + go p.readStdout(stdout) + go p.wait() + return p, nil +} + func (p *process) initialize(ctx context.Context) ([]protocol.ChatModel, accountIdentity, error) { raw, err := p.request(ctx, "initialize", map[string]any{"subtype": "initialize"}) if err != nil { return nil, accountIdentity{}, fmt.Errorf("initialize Claude Code: %w%s", err, p.diagnostic()) } + return parseInitialize(raw) +} + +// parseInitialize reads the CLI's initialize response: its model catalog and +// signed-in account. +func parseInitialize(raw json.RawMessage) ([]protocol.ChatModel, accountIdentity, error) { var reply struct { Models []struct { Value string `json:"value"` @@ -237,7 +282,6 @@ func (p *process) write(value any) error { if err != nil { return err } - data = append(data, '\n') p.writeMu.Lock() defer p.writeMu.Unlock() p.mu.Lock() @@ -246,6 +290,10 @@ func (p *process) write(value any) error { if exited { return errProcessClosed } + if p.link != nil { + return p.link.send(claudehost.Frame{Op: claudehost.OpIn, Data: data}) + } + data = append(data, '\n') if _, err := p.stdin.Write(data); err != nil { return fmt.Errorf("write to Claude Code: %w", err) } @@ -314,7 +362,9 @@ func (p *process) finish(err error) { default: } } - _ = p.stdin.Close() + if p.stdin != nil { + _ = p.stdin.Close() + } p.session.processExit(p, err) } @@ -325,6 +375,10 @@ func (p *process) close() { if exited { return } + if p.link != nil { + p.closeHosted() + return + } _ = p.stdin.Close() if p.cmd.Process != nil { _ = p.cmd.Process.Signal(os.Interrupt) diff --git a/internal/claude/testdata/fake-claude.py b/internal/claude/testdata/fake-claude.py index 521f24a0..e551497c 100644 --- a/internal/claude/testdata/fake-claude.py +++ b/internal/claude/testdata/fake-claude.py @@ -72,6 +72,15 @@ def user_message(text): send({"type": "system", "subtype": "init", "session_id": state["session"], "model": "claude-test", "permissionMode": "default", "cwd": os.getcwd()}) message_id = "msg_fake_%d" % state["turn"] + if "gate" in text: + gated(message_id) + return + if "background" in text: + background(message_id) + return + if "notify" in text: + send({"type": "system", "subtype": "task_notification", "task_id": "b1", + "tool_use_id": "toolu_bg_fake", "status": "completed", "summary": "Background command completed (exit code 0)"}) send({"type": "stream_event", "event": {"type": "message_start", "message": {"id": message_id}}}) send({"type": "stream_event", "event": {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}}}) for delta in ("Hello ", "world"): @@ -125,6 +134,46 @@ def user_message(text): finish(message_id) +def wait_for(path): + import time + while not os.path.exists(path): + time.sleep(0.02) + + +def gated(message_id): + """A turn that streams, then waits for FAKE_CLAUDE_GATE to exist before it + streams the rest and finishes: the test restarts the server in between.""" + gate = os.environ["FAKE_CLAUDE_GATE"] + send({"type": "stream_event", "event": {"type": "message_start", "message": {"id": message_id}}}) + send({"type": "stream_event", "event": {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}}}) + send({"type": "stream_event", "event": {"type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": "Working"}}}) + wait_for(gate) + send({"type": "stream_event", "event": {"type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": " after the restart"}}}) + send({"type": "stream_event", "event": {"type": "content_block_stop", "index": 0}}) + send({"type": "stream_event", "event": {"type": "message_stop"}}) + text = "Working after the restart" + append({"type": "assistant", "uuid": "asst-%d" % state["turn"], "timestamp": stamp(), + "message": {"id": message_id, "role": "assistant", "model": "claude-test", + "content": [{"type": "text", "text": text}]}}) + send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test", + "content": [{"type": "text", "text": text}]}}) + finish(message_id) + + +def background(message_id): + """Starts a background Bash command that a later "notify" turn finishes.""" + tool_id = "toolu_bg_fake" + use = {"type": "tool_use", "id": tool_id, "name": "Bash", + "input": {"command": "sleep 600", "run_in_background": True}} + send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}}) + block = {"type": "tool_result", "tool_use_id": tool_id, + "content": "Command running in background with ID: b1. Output is being written to: /tmp/b1.output"} + send({"type": "user", "message": {"role": "user", "content": [block]}}) + finish(message_id) + + def finish(message_id): # Shape of SDKRateLimitEvent in Claude Code 2.1.316 (synthetic numbers). if os.environ.get("FAKE_CLAUDE_RATE_LIMIT"): diff --git a/internal/claude/translate.go b/internal/claude/translate.go index c017c529..9bc02c22 100644 --- a/internal/claude/translate.go +++ b/internal/claude/translate.go @@ -348,6 +348,9 @@ func (s *session) claimStreamedLocked(kind, full string, local int) (blockState, // broadcastLocked emits while the session lock is already held: the caller is a // stream-delta path that owns state it just updated. func (s *session) broadcastLocked(frame protocol.ChatEventFrame) { + if s.silent { + return + } targets := make([]chan protocol.ChatEventFrame, 0, len(s.subs)) for ch := range s.subs { targets = append(targets, ch) @@ -568,6 +571,9 @@ func (s *session) handleResult(envelope lineEnvelope) { turn := s.turnID running := s.running proc := s.proc + if s.silent { + proc = nil // replayed: the live usage query is made once caught up + } s.running = false s.turnID = "" // Messages written mid-turn that the CLI did not fold into this turn run diff --git a/internal/claudehost/client.go b/internal/claudehost/client.go new file mode 100644 index 00000000..7e740bc0 --- /dev/null +++ b/internal/claudehost/client.go @@ -0,0 +1,217 @@ +package claudehost + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "os/exec" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +// Conn is a server's connection to one host. +type Conn struct { + conn net.Conn + reader *bufio.Reader + writeMu sync.Mutex +} + +// Dial connects to a host socket. +func Dial(socket string, timeout time.Duration) (*Conn, error) { + conn, err := net.DialTimeout("unix", socket, timeout) + if err != nil { + return nil, err + } + return &Conn{conn: conn, reader: bufio.NewReaderSize(conn, 256<<10)}, nil +} + +// Send writes one frame. +func (c *Conn) Send(frame Frame) error { + data, err := json.Marshal(frame) + if err != nil { + return err + } + data = append(data, '\n') + c.writeMu.Lock() + defer c.writeMu.Unlock() + _, err = c.conn.Write(data) + return err +} + +// Recv reads one frame. +func (c *Conn) Recv() (Frame, error) { + line, err := c.reader.ReadBytes('\n') + if err != nil && len(strings.TrimSpace(string(line))) == 0 { + return Frame{}, err + } + var frame Frame + if err := json.Unmarshal(line, &frame); err != nil { + return Frame{}, fmt.Errorf("invalid host frame: %w", err) + } + return frame, nil +} + +// Buffered reports whether more frames are already read from the socket. +func (c *Conn) Buffered() bool { return c.reader.Buffered() > 0 } + +func (c *Conn) Close() error { return c.conn.Close() } + +// Attach dials socket and starts delivery from offset. It returns the status +// the host replied with; output frames follow on the connection. +func Attach(socket string, offset int64, timeout time.Duration) (*Conn, *Status, error) { + conn, err := Dial(socket, timeout) + if err != nil { + return nil, nil, err + } + _ = conn.conn.SetDeadline(time.Now().Add(timeout)) + if err := conn.Send(Frame{Op: OpAttach, Offset: offset}); err != nil { + conn.Close() + return nil, nil, err + } + frame, err := conn.Recv() + if err != nil { + conn.Close() + return nil, nil, err + } + _ = conn.conn.SetDeadline(time.Time{}) + if frame.Op != OpStatus || frame.Status == nil { + conn.Close() + return nil, nil, fmt.Errorf("unexpected host reply %q", frame.Op) + } + return conn, frame.Status, nil +} + +// SpawnOptions starts one host. +type SpawnOptions struct { + // HostBinary is the pipelines-claude-host executable. + HostBinary string + // HostArgs are extra arguments placed before the host flags (a test binary + // uses them to select host mode). + HostArgs []string + Dir string + SessionID string + Claude string + Args []string + Env []string + Cwd string + Idle time.Duration +} + +// Spawn starts a host in its own session, so it outlives the caller, and +// waits until its socket is serving. The caller reaps the returned command. +func Spawn(ctx context.Context, options SpawnOptions) (*exec.Cmd, error) { + if err := EnsureDir(options.Dir); err != nil { + return nil, err + } + paths := PathsFor(options.Dir, options.SessionID) + args := append([]string(nil), options.HostArgs...) + args = append(args, "-dir", options.Dir, "-id", options.SessionID) + if options.Idle > 0 { + args = append(args, "-idle", options.Idle.String()) + } + args = append(args, "--", options.Claude) + args = append(args, options.Args...) + cmd := exec.Command(options.HostBinary, args...) + cmd.Dir = options.Cwd + cmd.Env = options.Env + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + logFile, err := os.OpenFile(paths.Log, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return nil, err + } + defer logFile.Close() + cmd.Stderr = logFile + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start the Claude Code host: %w", err) + } + type ready struct { + line string + err error + } + result := make(chan ready, 1) + go func() { + line, err := bufio.NewReader(stdout).ReadString('\n') + result <- ready{line: strings.TrimSpace(line), err: err} + }() + timer := time.NewTimer(15 * time.Second) + defer timer.Stop() + fail := func(err error) (*exec.Cmd, error) { + _ = cmd.Process.Kill() + _ = cmd.Wait() + return nil, err + } + select { + case <-ctx.Done(): + return fail(ctx.Err()) + case <-timer.C: + return fail(errors.New("the Claude Code host did not start in time")) + case r := <-result: + if strings.HasPrefix(r.line, "ready ") { + return cmd, nil + } + message := strings.TrimPrefix(r.line, "error ") + if message == "" { + message = "the Claude Code host exited during start" + if data, err := os.ReadFile(paths.Log); err == nil { + if tail := strings.TrimSpace(string(data)); tail != "" { + if len(tail) > 400 { + tail = tail[len(tail)-400:] + } + message += ": " + tail + } + } + } + return fail(errors.New(message)) + } +} + +// ProcessMatches reports whether pid is alive and its command line mentions +// every needle, so a recorded pid that the system reused is never signalled. +func ProcessMatches(pid int, needles ...string) bool { + if pid <= 1 { + return false + } + if syscall.Kill(pid, 0) != nil { + return false + } + out, err := exec.Command("ps", "-o", "command=", "-p", strconv.Itoa(pid)).Output() + if err != nil { + return false + } + command := string(out) + for _, needle := range needles { + if !strings.Contains(command, needle) { + return false + } + } + return true +} + +// KillStale stops what a dead or unreachable host left behind: the host +// itself and its CLI's process group, each only when its command line still +// names the session. It then removes the session's host files. +func KillStale(dir string, record Record) { + if ProcessMatches(record.HostPID, record.SessionID) { + _ = syscall.Kill(record.HostPID, syscall.SIGKILL) + } + if ProcessMatches(record.ClaudePID, record.SessionID) { + _ = syscall.Kill(-record.ClaudePID, syscall.SIGTERM) + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && syscall.Kill(record.ClaudePID, 0) == nil { + time.Sleep(50 * time.Millisecond) + } + _ = syscall.Kill(-record.ClaudePID, syscall.SIGKILL) + } + Remove(dir, record.SessionID) +} diff --git a/internal/claudehost/host.go b/internal/claudehost/host.go new file mode 100644 index 00000000..0235ea97 --- /dev/null +++ b/internal/claudehost/host.go @@ -0,0 +1,710 @@ +package claudehost + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "net" + "os" + "os/exec" + "os/signal" + "sort" + "strings" + "sync" + "syscall" + "time" +) + +// Defaults for a host started without explicit limits. +const ( + DefaultIdle = 20 * time.Minute + DefaultMaxDetached = 24 * time.Hour + DefaultSpoolLimit = 64 << 20 + maxLineBytes = 32 << 20 + stderrRetained = 8 << 10 +) + +type lineRef struct{ start, end int64 } + +type transition struct { + offset int64 + busy bool +} + +type client struct { + conn net.Conn + writeMu sync.Mutex + gone chan struct{} + once sync.Once + exitSent bool +} + +func (c *client) send(frame Frame) error { + data, err := json.Marshal(frame) + if err != nil { + return err + } + data = append(data, '\n') + c.writeMu.Lock() + defer c.writeMu.Unlock() + _, err = c.conn.Write(data) + return err +} + +func (c *client) close() { + c.once.Do(func() { + close(c.gone) + _ = c.conn.Close() + }) +} + +type host struct { + dir string + id string + paths Paths + idle time.Duration + maxDetached time.Duration + spoolLimit int64 + listener net.Listener + + cmd *exec.Cmd + stdin io.WriteCloser + stdinMu sync.Mutex + + fileMu sync.RWMutex // guards spool replacement against readers + mu sync.Mutex + spool *os.File + base int64 + end int64 + lines []lineRef + acked int64 + trans []transition + busy bool + // pending maps unanswered CLI control request ids to their offsets. + pending map[string]int64 + initID string + init json.RawMessage + modelReqs map[string]string + modeReqs map[string]string + model string + mode string + exited bool + exitErr string + exitedAt time.Time + stderr strings.Builder + changed chan struct{} + active *client + detached time.Time + activity time.Time + stopping bool + stopped chan struct{} + claudeEnd chan struct{} +} + +// Main runs a host. args are the host flags, then "--", then the Claude Code +// executable and its arguments. The working directory and environment are the +// CLI's own. It returns the process exit code. +func Main(args []string) int { + flags := flag.NewFlagSet("pipelines-claude-host", flag.ContinueOnError) + dir := flags.String("dir", "", "private host directory (registry, socket, spool)") + id := flags.String("id", "", "Claude session id") + idle := flags.Duration("idle", DefaultIdle, "release an idle CLI after this long with no server attached") + maxDetached := flags.Duration("max-detached", DefaultMaxDetached, "release the CLI after this long with no server attached, even when busy") + spoolLimit := flags.Int64("spool-limit", DefaultSpoolLimit, "bytes of acknowledged output kept in the spool") + if err := flags.Parse(args); err != nil { + return 2 + } + command := flags.Args() + if len(command) == 0 || *id == "" || *dir == "" { + fmt.Fprintln(os.Stderr, "usage: pipelines-claude-host -dir DIR -id SESSION -- claude ARGS...") + return 2 + } + log.SetFlags(log.LstdFlags) + log.SetPrefix("claude-host " + *id + ": ") + syscall.Umask(0o077) + if err := EnsureDir(*dir); err != nil { + log.Print(err) + return 1 + } + h := &host{ + dir: *dir, id: *id, paths: PathsFor(*dir, *id), idle: *idle, maxDetached: *maxDetached, spoolLimit: *spoolLimit, + pending: map[string]int64{}, modelReqs: map[string]string{}, modeReqs: map[string]string{}, + changed: make(chan struct{}), stopped: make(chan struct{}), claudeEnd: make(chan struct{}), + trans: []transition{{offset: 0, busy: false}}, activity: time.Now(), detached: time.Now(), + } + h.model, h.mode = flagValue(command, "--model"), flagValue(command, "--permission-mode") + if err := h.start(command); err != nil { + log.Print(err) + fmt.Fprintln(os.Stdout, "error "+strings.ReplaceAll(err.Error(), "\n", " ")) + return 1 + } + // The server waits for this line, then stops reading our stdout. + fmt.Fprintln(os.Stdout, "ready "+h.paths.Socket) + if null, err := os.OpenFile(os.DevNull, os.O_RDWR, 0); err == nil { + _ = syscall.Dup2(int(null.Fd()), 1) + _ = syscall.Dup2(int(null.Fd()), 0) + _ = null.Close() + } + signals := make(chan os.Signal, 2) + signal.Ignore(syscall.SIGHUP, syscall.SIGPIPE) + signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT) + go func() { + <-signals + h.shutdown("signal") + }() + go h.accept() + go h.watchIdle() + <-h.stopped + return 0 +} + +func flagValue(args []string, name string) string { + for index, arg := range args { + if arg == name && index+1 < len(args) { + return args[index+1] + } + if strings.HasPrefix(arg, name+"=") { + return strings.TrimPrefix(arg, name+"=") + } + } + return "" +} + +func (h *host) start(command []string) error { + if conn, err := net.DialTimeout("unix", h.paths.Socket, time.Second); err == nil { + _ = conn.Close() + return errors.New("a host is already serving this session") + } + _ = os.Remove(h.paths.Socket) + spool, err := os.OpenFile(h.paths.Spool, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o600) + if err != nil { + return err + } + h.spool = spool + listener, err := net.Listen("unix", h.paths.Socket) + if err != nil { + return err + } + _ = os.Chmod(h.paths.Socket, 0o600) + h.listener = listener + cmd := exec.Command(command[0], command[1:]...) + cmd.Env = os.Environ() + // Its own process group, so the CLI and its tool subprocesses stop together. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + _ = listener.Close() + _ = os.Remove(h.paths.Socket) + return fmt.Errorf("start Claude Code: %w", err) + } + h.cmd, h.stdin = cmd, stdin + cwd, _ := os.Getwd() + record := Record{Version: Version, SessionID: h.id, HostPID: os.Getpid(), ClaudePID: cmd.Process.Pid, + Socket: h.paths.Socket, Spool: h.paths.Spool, Directory: cwd, Created: time.Now().UnixMilli()} + data, _ := json.Marshal(record) + if err := writeFileAtomic(h.paths.Record, data); err != nil { + _ = cmd.Process.Kill() + return err + } + stderrDone := make(chan struct{}) + go h.readStderr(stderr, stderrDone) + readDone := make(chan struct{}) + go h.readStdout(stdout, readDone) + go func() { + err := cmd.Wait() + <-readDone + select { + case <-stderrDone: + case <-time.After(time.Second): + } + h.mu.Lock() + h.exited = true + h.exitedAt = time.Now() + if err != nil { + h.exitErr = err.Error() + } + h.broadcastLocked() + h.mu.Unlock() + close(h.claudeEnd) + }() + return nil +} + +func (h *host) readStderr(stderr io.Reader, done chan struct{}) { + defer close(done) + buffer := make([]byte, 4<<10) + for { + n, err := stderr.Read(buffer) + if n > 0 { + h.mu.Lock() + if remaining := stderrRetained - h.stderr.Len(); remaining > 0 { + if n > remaining { + n = remaining + } + h.stderr.Write(buffer[:n]) + } + h.mu.Unlock() + } + if err != nil { + return + } + } +} + +func (h *host) readStdout(stdout io.Reader, done chan struct{}) { + defer close(done) + reader := bufio.NewReaderSize(stdout, 64<<10) + for { + line, err := reader.ReadBytes('\n') + if len(line) > 0 && len(line) <= maxLineBytes { + trimmed := strings.TrimSpace(string(line)) + if trimmed != "" { + h.appendLine([]byte(trimmed)) + } + } + if err != nil { + return + } + } +} + +// broadcastLocked wakes every sender. Callers hold h.mu. +func (h *host) broadcastLocked() { + close(h.changed) + h.changed = make(chan struct{}) +} + +type snoop struct { + Type string `json:"type"` + Subtype string `json:"subtype"` + RequestID string `json:"request_id"` + ParentToolUseID string `json:"parent_tool_use_id"` + Request struct { + Subtype string `json:"subtype"` + Model string `json:"model"` + Mode string `json:"mode"` + } `json:"request"` + Response struct { + Subtype string `json:"subtype"` + RequestID string `json:"request_id"` + Response json.RawMessage `json:"response"` + } `json:"response"` +} + +func (h *host) setBusyLocked(busy bool, offset int64) { + if h.busy == busy { + return + } + h.busy = busy + h.trans = append(h.trans, transition{offset: offset, busy: busy}) +} + +func (h *host) busyAtLocked(offset int64) bool { + busy := false + for _, t := range h.trans { + if t.offset > offset { + break + } + busy = t.busy + } + return busy +} + +func (h *host) appendLine(line []byte) { + var env snoop + _ = json.Unmarshal(line, &env) + h.mu.Lock() + defer h.mu.Unlock() + start := h.end + data := append(append([]byte(nil), line...), '\n') + h.fileMu.RLock() + _, err := h.spool.WriteAt(data, start-h.base) + h.fileMu.RUnlock() + if err != nil { + log.Printf("spool write: %v", err) + } + h.end += int64(len(data)) + h.lines = append(h.lines, lineRef{start: start, end: h.end}) + h.activity = time.Now() + switch env.Type { + case "control_request": + if env.RequestID != "" { + h.pending[env.RequestID] = start + } + case "control_cancel_request": + delete(h.pending, env.RequestID) + case "control_response": + id := env.Response.RequestID + if env.Response.Subtype == "success" { + if id != "" && id == h.initID { + h.init = append(json.RawMessage(nil), env.Response.Response...) + } + if model, ok := h.modelReqs[id]; ok { + h.model = model + } + if mode, ok := h.modeReqs[id]; ok { + h.mode = mode + } + } + delete(h.modelReqs, id) + delete(h.modeReqs, id) + case "result": + h.setBusyLocked(false, h.end) + // A finished turn no longer waits on any approval. + h.pending = map[string]int64{} + case "stream_event", "assistant": + if env.ParentToolUseID == "" { + h.setBusyLocked(true, start) + } + } + h.compactLocked() + h.broadcastLocked() +} + +// compactLocked bounds the spool. Acknowledged output beyond the limit is +// dropped first; unacknowledged output is dropped only beyond four times the +// limit (a server gone for very long), which a reconnecting server sees as a +// gap because Acked < Base. +func (h *host) compactLocked() { + size := h.end - h.base + if size <= h.spoolLimit { + return + } + keep := h.acked + if h.end-keep > 4*h.spoolLimit { + keep = h.end - h.spoolLimit + } + index := sort.Search(len(h.lines), func(i int) bool { return h.lines[i].start >= keep }) + if index == 0 || index >= len(h.lines) { + return + } + keep = h.lines[index].start + if keep-h.base < size/4 { + return // not worth rewriting yet + } + next, err := os.OpenFile(h.paths.Spool+".next", os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o600) + if err != nil { + return + } + h.fileMu.Lock() + defer h.fileMu.Unlock() + if _, err := io.Copy(next, io.NewSectionReader(h.spool, keep-h.base, h.end-keep)); err != nil { + _ = next.Close() + _ = os.Remove(h.paths.Spool + ".next") + return + } + if err := os.Rename(h.paths.Spool+".next", h.paths.Spool); err != nil { + _ = next.Close() + return + } + _ = h.spool.Close() + h.spool = next + h.base = keep + h.lines = append([]lineRef(nil), h.lines[index:]...) +} + +func (h *host) statusLocked() *Status { + pending := make([]Pending, 0, len(h.pending)) + for id, offset := range h.pending { + pending = append(pending, Pending{ID: id, Offset: offset}) + } + sort.Slice(pending, func(i, j int) bool { return pending[i].Offset < pending[j].Offset }) + cwd, _ := os.Getwd() + status := &Status{ + Version: Version, SessionID: h.id, HostPID: os.Getpid(), + Base: h.base, End: h.end, Acked: h.acked, Busy: h.busy, BusyAtAcked: h.busyAtLocked(h.acked), + Init: h.init, Model: h.model, Mode: h.mode, Pending: pending, + Exited: h.exited, ExitErr: h.exitErr, Stderr: h.stderr.String(), Directory: cwd, + } + if h.cmd != nil && h.cmd.Process != nil { + status.ClaudePID = h.cmd.Process.Pid + } + return status +} + +func (h *host) accept() { + for { + conn, err := h.listener.Accept() + if err != nil { + return + } + c := &client{conn: conn, gone: make(chan struct{})} + go h.serveClient(c) + } +} + +func (h *host) serveClient(c *client) { + defer h.dropClient(c) + reader := bufio.NewReaderSize(c.conn, 64<<10) + for { + line, err := reader.ReadBytes('\n') + if len(strings.TrimSpace(string(line))) > 0 { + var frame Frame + if json.Unmarshal(line, &frame) != nil { + _ = c.send(Frame{Op: OpError, Error: "invalid frame"}) + } else if !h.handle(c, frame) { + return + } + } + if err != nil { + return + } + } +} + +func (h *host) dropClient(c *client) { + c.close() + h.mu.Lock() + if h.active == c { + h.active = nil + h.detached = time.Now() + } + h.mu.Unlock() +} + +// handle runs one client frame; false ends the connection. +func (h *host) handle(c *client, frame Frame) bool { + switch frame.Op { + case OpAttach: + h.mu.Lock() + previous := h.active + h.active = c + status := h.statusLocked() + h.mu.Unlock() + if previous != nil && previous != c { + previous.close() + } + if err := c.send(Frame{Op: OpStatus, Status: status}); err != nil { + return false + } + go h.deliver(c, frame.Offset) + case OpIn: + h.writeIn(frame.Data) + case OpAck: + h.mu.Lock() + if frame.Offset > h.acked && frame.Offset <= h.end { + h.acked = frame.Offset + h.trimTransitionsLocked() + } + h.mu.Unlock() + case OpStatus: + h.mu.Lock() + status := h.statusLocked() + h.mu.Unlock() + _ = c.send(Frame{Op: OpStatus, Status: status}) + case OpClose: + h.stopClaude() + _ = c.send(Frame{Op: OpClosed}) + h.shutdown("closed") + return false + default: + _ = c.send(Frame{Op: OpError, Error: "unknown op " + frame.Op}) + } + return true +} + +func (h *host) trimTransitionsLocked() { + keep := 0 + for i, t := range h.trans { + if t.offset <= h.acked { + keep = i + } + } + h.trans = append([]transition(nil), h.trans[keep:]...) +} + +func (h *host) writeIn(data json.RawMessage) { + if len(data) == 0 { + return + } + var env snoop + _ = json.Unmarshal(data, &env) + h.mu.Lock() + h.activity = time.Now() + switch env.Type { + case "user": + h.setBusyLocked(true, h.end) + case "control_response": + delete(h.pending, env.Response.RequestID) + case "control_request": + switch env.Request.Subtype { + case "initialize": + h.initID = env.RequestID + case "set_model": + h.modelReqs[env.RequestID] = env.Request.Model + case "set_permission_mode": + h.modeReqs[env.RequestID] = env.Request.Mode + } + } + h.mu.Unlock() + line := append(append([]byte(nil), data...), '\n') + h.stdinMu.Lock() + defer h.stdinMu.Unlock() + if _, err := h.stdin.Write(line); err != nil { + log.Printf("write to Claude Code: %v", err) + } +} + +// deliver streams spooled lines from offset, then the exit, to one client. +func (h *host) deliver(c *client, offset int64) { + cursor := offset + for { + h.mu.Lock() + if h.active != c { + h.mu.Unlock() + return + } + index := sort.Search(len(h.lines), func(i int) bool { return h.lines[i].start >= cursor }) + if index < len(h.lines) { + ref := h.lines[index] + h.fileMu.RLock() + h.mu.Unlock() + data := make([]byte, ref.end-ref.start) + _, err := h.spool.ReadAt(data, ref.start-h.base) + h.fileMu.RUnlock() + if err != nil { + log.Printf("spool read: %v", err) + c.close() + return + } + if err := c.send(Frame{Op: OpOut, Offset: ref.start, End: ref.end, Data: json.RawMessage(strings.TrimSpace(string(data)))}); err != nil { + c.close() + return + } + cursor = ref.end + continue + } + if h.exited && !c.exitSent { + c.exitSent = true + status := h.statusLocked() + h.mu.Unlock() + _ = c.send(Frame{Op: OpExit, Status: status, Error: status.ExitErr}) + // The server has everything; it closes the connection, then we go. + go func() { + select { + case <-c.gone: + case <-time.After(30 * time.Second): + } + h.shutdown("exited") + }() + continue + } + changed := h.changed + h.mu.Unlock() + select { + case <-changed: + case <-c.gone: + return + case <-h.stopped: + return + } + } +} + +// stopClaude ends the CLI: stdin EOF, then an interrupt, then a kill, to the +// CLI's whole process group. +func (h *host) stopClaude() { + if h.cmd == nil || h.cmd.Process == nil { + return + } + h.stdinMu.Lock() + _ = h.stdin.Close() + h.stdinMu.Unlock() + pid := h.cmd.Process.Pid + select { + case <-h.claudeEnd: + return + case <-time.After(500 * time.Millisecond): + } + _ = syscall.Kill(-pid, syscall.SIGINT) + select { + case <-h.claudeEnd: + return + case <-time.After(3 * time.Second): + } + _ = syscall.Kill(-pid, syscall.SIGKILL) + select { + case <-h.claudeEnd: + case <-time.After(2 * time.Second): + } +} + +// shutdown stops the CLI, removes the host files and ends Main. +func (h *host) shutdown(reason string) { + h.mu.Lock() + if h.stopping { + h.mu.Unlock() + return + } + h.stopping = true + active := h.active + h.mu.Unlock() + log.Printf("stopping (%s)", reason) + h.stopClaude() + _ = h.listener.Close() + Remove(h.dir, h.id) + _ = os.Remove(h.paths.Spool + ".next") + if active != nil { + active.close() + } + close(h.stopped) +} + +// watchIdle releases the CLI while no server is attached: after the idle +// expiry when nothing is in flight (the same release the server applies), and +// after maxDetached regardless. A CLI that exited with nobody attached is +// cleaned up after the idle expiry. +func (h *host) watchIdle() { + tick := h.idle / 10 + if tick > 5*time.Second { + tick = 5 * time.Second + } + if tick < 50*time.Millisecond { + tick = 50 * time.Millisecond + } + ticker := time.NewTicker(tick) + defer ticker.Stop() + for { + select { + case <-h.stopped: + return + case <-ticker.C: + } + now := time.Now() + h.mu.Lock() + attached := h.active != nil + idle := !h.busy && len(h.pending) == 0 && now.Sub(h.detached) > h.idle && now.Sub(h.activity) > h.idle + tooLong := now.Sub(h.detached) > h.maxDetached + exitedLong := h.exited && now.Sub(h.exitedAt) > h.idle + h.mu.Unlock() + if attached { + continue + } + switch { + case exitedLong: + h.shutdown("exited with no server attached") + return + case idle: + h.shutdown("idle with no server attached") + return + case tooLong: + h.shutdown("no server attached for " + h.maxDetached.String()) + return + } + } +} diff --git a/internal/claudehost/wire.go b/internal/claudehost/wire.go new file mode 100644 index 00000000..3482980d --- /dev/null +++ b/internal/claudehost/wire.go @@ -0,0 +1,180 @@ +// Package claudehost keeps one Claude Code CLI process alive across Pipelines +// server restarts. +// +// The server cannot own a `claude -p` process directly and also restart: the +// CLI's stdin/stdout pipes die with their parent. A host is a small helper +// process in its own session that owns those pipes instead. It spools the +// CLI's stream-json output to a bounded per-session file with byte offsets and +// serves one local Unix socket (0600, in a 0700 directory) on which the server +// writes stdin frames, reads output from an offset, acknowledges what it has +// processed, and finally closes the host. A restarted server finds live hosts +// through their registry files, reconnects and replays from its last +// acknowledged offset. +// +// The wire format is one JSON object per line in both directions. +package claudehost + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" +) + +// Version is the host protocol version reported in every status. +const Version = 1 + +// Client-to-host operations. +const ( + // OpAttach starts (or restarts) delivery from Frame.Offset. The host + // replies with OpStatus, then streams OpOut frames. A newer attach replaces + // an older connection. + OpAttach = "attach" + // OpIn writes Frame.Data, one JSON line, to the CLI's stdin. + OpIn = "in" + // OpAck commits that output up to Frame.Offset has been processed. + OpAck = "ack" + // OpStatus asks for a status snapshot without changing delivery. + OpStatus = "status" + // OpClose stops the CLI and the host. The host replies OpClosed. + OpClose = "close" +) + +// Host-to-client operations. +const ( + OpOut = "out" + OpExit = "exit" + OpClosed = "closed" + OpError = "error" +) + +// Frame is one line on the socket. +type Frame struct { + Op string `json:"op"` + Offset int64 `json:"offset,omitempty"` + End int64 `json:"end,omitempty"` + Data json.RawMessage `json:"data,omitempty"` + Status *Status `json:"status,omitempty"` + Error string `json:"error,omitempty"` +} + +// Pending is a CLI control request (can_use_tool, a hook) that nobody has +// answered yet. It survives a disconnected server. +type Pending struct { + ID string `json:"id"` + Offset int64 `json:"offset"` +} + +// Status is the host's snapshot of its CLI. +type Status struct { + Version int `json:"version"` + SessionID string `json:"session_id"` + HostPID int `json:"host_pid"` + ClaudePID int `json:"claude_pid"` + // Base is the oldest offset still in the spool; End is the offset after the + // last spooled line; Acked is the last offset a client committed. + Base int64 `json:"base"` + End int64 `json:"end"` + Acked int64 `json:"acked"` + // Busy is whether a turn is in flight now; BusyAtAcked is whether one was + // in flight at the Acked offset, which is where a reconnecting server's + // live processing resumes. + Busy bool `json:"busy"` + BusyAtAcked bool `json:"busy_at_acked"` + // Init is the CLI's initialize response (models, account), so a + // reconnecting server does not initialize the CLI twice. + Init json.RawMessage `json:"init,omitempty"` + // Model and Mode are the CLI's model and permission mode as last applied + // (start flags, then successful set_model / set_permission_mode). + Model string `json:"model,omitempty"` + Mode string `json:"mode,omitempty"` + Pending []Pending `json:"pending"` + Exited bool `json:"exited,omitempty"` + ExitErr string `json:"exit_error,omitempty"` + Stderr string `json:"stderr,omitempty"` + // Directory is the CLI's working directory. + Directory string `json:"directory,omitempty"` +} + +// Record is the registry file a host writes next to its socket so a +// restarted server can find it. +type Record struct { + Version int `json:"version"` + SessionID string `json:"session_id"` + HostPID int `json:"host_pid"` + ClaudePID int `json:"claude_pid"` + Socket string `json:"socket"` + Spool string `json:"spool"` + Directory string `json:"directory"` + Created int64 `json:"created"` +} + +// Key is the short file name used for a session's host files. Unix socket +// paths are limited to about 100 bytes, so the session id is hashed. +func Key(sessionID string) string { + sum := sha256.Sum256([]byte(sessionID)) + return hex.EncodeToString(sum[:8]) +} + +// Paths are one session's files inside a host directory. +type Paths struct { + Socket, Record, Spool, Log string +} + +func PathsFor(dir, sessionID string) Paths { + key := Key(sessionID) + return Paths{ + Socket: filepath.Join(dir, key+".sock"), + Record: filepath.Join(dir, key+".json"), + Spool: filepath.Join(dir, key+".spool"), + Log: filepath.Join(dir, key+".log"), + } +} + +// EnsureDir creates a private host directory. +func EnsureDir(dir string) error { + if strings.TrimSpace(dir) == "" || !filepath.IsAbs(dir) { + return errors.New("the Claude host directory must be absolute") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + return os.Chmod(dir, 0o700) +} + +// Records lists the registry files in dir. Unreadable files are skipped. +func Records(dir string) []Record { + matches, _ := filepath.Glob(filepath.Join(dir, "*.json")) + records := make([]Record, 0, len(matches)) + for _, path := range matches { + data, err := os.ReadFile(path) + if err != nil { + continue + } + var record Record + if json.Unmarshal(data, &record) != nil || record.SessionID == "" || record.Socket == "" { + continue + } + records = append(records, record) + } + return records +} + +// Remove deletes one session's host files. +func Remove(dir, sessionID string) { + paths := PathsFor(dir, sessionID) + for _, path := range []string{paths.Record, paths.Socket, paths.Spool, paths.Log} { + _ = os.Remove(path) + } +} + +func writeFileAtomic(path string, data []byte) error { + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/internal/codex/chat.go b/internal/codex/chat.go index 4e878c5c..13de0c1a 100644 --- a/internal/codex/chat.go +++ b/internal/codex/chat.go @@ -176,13 +176,13 @@ func (c *Chat) Close() { _ = c.conn.Close(websocket.StatusNormalClosure, "") } -// DetachSessions is the server-restart shutdown: threads and their turns live +// Detach is the server-restart shutdown (chatShutdownDetacher): threads and their turns live // in the Codex App Server, not in this client, so closing the control // connection leaves them running; no turn is interrupted. The next server // process reconnects and reconciles each thread through thread/read. An // approval or question the App Server sent to this connection is bound to it, // so one still pending at shutdown is not answerable after the restart. -func (c *Chat) DetachSessions() { c.Close() } +func (c *Chat) Detach() { c.Close() } func (c *Chat) readLoop() { for { diff --git a/internal/codex/detach_test.go b/internal/codex/detach_test.go index f47c8ac8..74b5b9c8 100644 --- a/internal/codex/detach_test.go +++ b/internal/codex/detach_test.go @@ -11,7 +11,7 @@ import ( // A server restart detaches from the App Server without interrupting the // thread's turn; the next server's fresh connection still sees the turn in // progress, then completed, through the read-only turn list. -func TestDetachSessionsLeavesTurnRunningForNextConnection(t *testing.T) { +func TestDetachLeavesTurnRunningForNextConnection(t *testing.T) { var completed atomic.Bool var mu sync.Mutex var methods []string @@ -29,7 +29,7 @@ func TestDetachSessionsLeavesTurnRunningForNextConnection(t *testing.T) { if running, err := before.ProbeTurn(context.Background(), fixtureThread); err != nil || !running { t.Fatalf("turn not running before the restart: %v %v", running, err) } - before.DetachSessions() + before.Detach() after := modelTestChat(t, appServer) if running, err := after.ProbeTurn(context.Background(), fixtureThread); err != nil || !running { t.Fatalf("turn did not survive the restart: %v %v", running, err) diff --git a/internal/installation/uninstall.go b/internal/installation/uninstall.go index bdc3ea22..c27634bd 100644 --- a/internal/installation/uninstall.go +++ b/internal/installation/uninstall.go @@ -40,7 +40,7 @@ func Uninstall(executable string, args []string, in io.Reader, out io.Writer) er } dir := filepath.Dir(resolved) var files []string - for _, command := range []string{"pipelines", "pipelines-server", "pipelines-tui"} { + for _, command := range []string{"pipelines", "pipelines-server", "pipelines-tui", "pipelines-claude-host"} { path := filepath.Join(dir, command) info, err := buildinfo.ReadFile(path) if os.IsNotExist(err) { diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index d8b00f58..9967eaf5 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -304,12 +304,12 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error { return nil } -// DetachSessions is the server-restart shutdown. Sessions, turns, questions, +// Detach is the server-restart shutdown (chatShutdownDetacher). Sessions, turns, questions, // permissions and shells live in the OpenCode service, not in this adapter, so // there is nothing to stop: the connection's owner decides whether the service // itself outlives the server (an attached endpoint and the detached owned chat // service do; a V1 serve child does not). -func (c *Chat) DetachSessions() {} +func (c *Chat) Detach() {} // Messages reads one session's stored turns. On the owned serve backend it // reuses Connection.Messages, including its read-only OutputFormat readback diff --git a/internal/server/chat_close.go b/internal/server/chat_close.go index e15695b3..2ea7091d 100644 --- a/internal/server/chat_close.go +++ b/internal/server/chat_close.go @@ -2,6 +2,7 @@ package server import ( "context" + "log" "strings" "git.a-wan.net/alan/pipelines/internal/pipeline" @@ -58,6 +59,13 @@ func handleChatClose(ctx context.Context, request protocol.Message, broker chatB } // A closed chat takes no more turns: prompts it still held are dropped. hub.clearQueue(ctx, sessionID) + // A harness that keeps the agent alive across restarts (Claude Code's + // host) stops it now; the transcript stays and a later prompt resumes it. + if releaser, ok := broker.(chatSessionReleaser); ok { + if err := releaser.ReleaseSession(ctx, sessionID); err != nil { + log.Printf("chat session %s: release after close: %v", sessionID, err) + } + } session, found, err := registry.OperatorSession(ctx, sessionID) if err != nil { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) diff --git a/internal/server/chat_detach.go b/internal/server/chat_detach.go index 9f1093be..6b57858d 100644 --- a/internal/server/chat_detach.go +++ b/internal/server/chat_detach.go @@ -13,23 +13,20 @@ import ( // Chat sessions survive a Pipelines restart. Only an operator action (closing // the chat, or Stop on its turn) stops one. // -// Per-backend contract. A chat backend whose sessions can outlive the server -// process implements chatSessionDetacher. On shutdown the server calls -// DetachSessions instead of Close: the backend releases its own connection or -// process handles but must not interrupt, abort or kill any session, turn, +// Per-backend contract (chatShutdownDetacher and chatSessionReleaser in +// chat_manager.go). On shutdown chatManager.Close calls Detach on a backend +// that implements it instead of Close: the backend releases its own connection +// or process handles but must not interrupt, abort or kill any session, turn, // question or background task. On the next start the server reattaches through // the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still // running?), Stream (re-subscribe), Permissions/Forms (pending questions and // approvals), Shells (background tasks) and Messages (read back a turn that -// ended while the server was down). A backend without DetachSessions is closed -// as before, and its sessions end with the server. +// ended while the server was down). chat_close calls ReleaseSession on a +// backend that keeps an agent alive until released. A backend without Detach +// is closed as before, and its sessions end with the server. // -// OpenCode (an attached or detached owned service) and Codex (its App Server) -// implement it; a Claude Code host that keeps its CLI processes alive can -// implement the same method. -type chatSessionDetacher interface { - DetachSessions() -} +// OpenCode (an attached or detached owned service), Codex (its App Server) and +// Claude Code (its detachable hosts) implement Detach. // chatLiveStore records, on shutdown, the chats left live on their harness so // the next start reattaches exactly those; *pipeline.Store implements it. diff --git a/internal/server/chat_manager.go b/internal/server/chat_manager.go index cbe86fbc..47143590 100644 --- a/internal/server/chat_manager.go +++ b/internal/server/chat_manager.go @@ -166,14 +166,32 @@ func (m *chatManager) harness(backend string) chatBroker { return client } +// chatShutdownDetacher is implemented by a backend whose live sessions can +// outlive this server: on shutdown it lets them go instead of stopping them, +// and the next server reattaches (Claude Code's detachable hosts). +type chatShutdownDetacher interface { + Detach() +} + +// chatSessionReleaser is implemented by a backend that keeps a session's agent +// running until it is explicitly released; chat_close releases it. +type chatSessionReleaser interface { + ReleaseSession(ctx context.Context, sessionID string) error +} + func closeHarness(client chatBroker) { + if detacher, ok := client.(chatShutdownDetacher); ok { + detacher.Detach() + return + } if closer, ok := client.(interface{ Close() }); ok { closer.Close() } } -// Close releases the owned Codex and Claude Code clients, if any, and stops -// later lazy attachment. +// Close lets go of the Codex and Claude Code clients, if any, and stops later +// lazy attachment. It runs at server shutdown: a backend whose sessions can +// survive it (chatShutdownDetacher) is detached, not stopped. func (m *chatManager) Close() { m.mu.Lock() m.closed = true @@ -183,26 +201,17 @@ func (m *chatManager) Close() { closeHarness(claudeClient) } -// DetachSessions is the shutdown counterpart of Close for a server that keeps -// chat sessions running across a restart (see chat_detach.go). Each harness -// that implements chatSessionDetacher releases its handles without stopping a -// session; one that does not is closed as before. The OpenCode connection's -// lifetime belongs to the caller (cmd/pipelines-server), so it is only detached. -func (m *chatManager) DetachSessions() { - m.mu.Lock() - m.closed = true - codexClient, claudeClient := m.codex, m.claude - m.mu.Unlock() - if detacher, ok := m.opencode.(chatSessionDetacher); ok { - detacher.DetachSessions() +// ReleaseSession releases a session's agent on its own harness after an +// explicit close. A harness without that notion is left as is. +func (m *chatManager) ReleaseSession(ctx context.Context, sessionID string) error { + adapter, err := m.brokerFor(ctx, sessionID) + if err != nil { + return nil } - for _, client := range []chatBroker{codexClient, claudeClient} { - if detacher, ok := client.(chatSessionDetacher); ok { - detacher.DetachSessions() - continue - } - closeHarness(client) + if releaser, ok := adapter.(chatSessionReleaser); ok { + return releaser.ReleaseSession(ctx, sessionID) } + return nil } // Backends lists the harnesses available for a new session. diff --git a/internal/server/restart_live_test.go b/internal/server/restart_live_test.go index 74bec5a2..402d0f90 100644 --- a/internal/server/restart_live_test.go +++ b/internal/server/restart_live_test.go @@ -29,8 +29,8 @@ import ( // PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m // // It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch -// database and a free loopback port (with only the operator's OpenCode -// provider login copied in), starts a chat turn and a one-stage run that each +// database and a free loopback port (with only the operator's opencode-go +// provider login row copied into the scratch OpenCode database), starts a chat turn and a one-stage run that each // run `sleep 30`, restarts the server mid-turn, and checks both finish // instead of being aborted. Every scratch process is killed afterwards. func TestRestartKeepsRealOpenCodeSessions(t *testing.T) { @@ -46,6 +46,9 @@ func TestRestartKeepsRealOpenCodeSessions(t *testing.T) { t.Fatal(err) } t.Logf("scratch root %s", root) + // Registered first, so it runs last: after the scratch processes are + // stopped, remove the tree (it holds a copied provider login row). + t.Cleanup(func() { _ = os.RemoveAll(root) }) home := filepath.Join(root, "home") env := scratchEnvironment(t, realHome, home) binary := filepath.Join(root, "pipelines-server") @@ -56,8 +59,14 @@ func TestRestartKeepsRealOpenCodeSessions(t *testing.T) { } // Chat creation activates the manager skill, so install the current // source's skills into the scratch harness roots. - skills := exec.Command("go", "run", "./cmd/pipelines", "skill", "install") - skills.Dir, skills.Env = filepath.Join("..", ".."), env + cli := filepath.Join(root, "pipelines") + buildCLI := exec.Command("go", "build", "-o", cli, "./cmd/pipelines") + buildCLI.Dir = filepath.Join("..", "..") + if out, err := buildCLI.CombinedOutput(); err != nil { + t.Fatalf("build CLI: %v\n%s", err, out) + } + skills := exec.Command(cli, "skill", "install") + skills.Env = env if out, err := skills.CombinedOutput(); err != nil { t.Fatalf("skill install: %v\n%s", err, out) } @@ -81,6 +90,10 @@ func TestRestartKeepsRealOpenCodeSessions(t *testing.T) { server.stop() killScratchServices(t, home) }) + // OpenCode 2.x keeps provider logins in its own database. The chat service + // has just created the scratch one; copy only the stage provider's login + // row into it before the stage service starts (scratch, owner-only). + copyProviderLogin(t, filepath.Join(realHome, ".local", "share", "opencode", "opencode.db"), filepath.Join(home, ".local", "share", "opencode", "opencode.db"), "opencode-go") ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) defer cancel() @@ -163,13 +176,6 @@ func scratchEnvironment(t *testing.T, realHome, home string) []string { t.Fatal(err) } } - auth, err := os.ReadFile(filepath.Join(realHome, ".local", "share", "opencode", "auth.json")) - if err != nil { - t.Skipf("no OpenCode login to copy: %v", err) - } - if err := os.WriteFile(filepath.Join(data, "opencode", "auth.json"), auth, 0600); err != nil { - t.Fatal(err) - } // A cold model catalog can leave the first service unable to resolve the // model; seed the public catalog cache (not a credential). if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil { @@ -197,6 +203,28 @@ func scratchEnvironment(t *testing.T, realHome, home string) []string { "PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off") } +func copyProviderLogin(t *testing.T, real, scratch, provider string) { + t.Helper() + if _, err := os.Stat(real); err != nil { + t.Skipf("no OpenCode login database: %v", err) + } + deadline := time.Now().Add(30 * time.Second) + for { + if _, err := os.Stat(scratch); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("scratch OpenCode database was not created") + } + time.Sleep(200 * time.Millisecond) + } + script := "ATTACH '" + real + "' AS real; INSERT OR REPLACE INTO credential SELECT * FROM real.credential WHERE integration_id='" + provider + "'; SELECT changes();" + out, err := exec.Command("sqlite3", "-bail", scratch, script).CombinedOutput() + if err != nil || strings.TrimSpace(string(out)) == "0" { + t.Skipf("could not copy the %s login into the scratch OpenCode database: %v %s", provider, err, out) + } +} + func opencodePath() string { if path, err := exec.LookPath("opencode"); err == nil { return path diff --git a/scripts/package.py b/scripts/package.py index 834e2876..39a47046 100755 --- a/scripts/package.py +++ b/scripts/package.py @@ -36,7 +36,7 @@ def package(output, version, target): if system not in ("darwin", "linux") or arch not in ("arm64", "amd64"): raise ValueError("Supported: source, darwin-arm64/amd64, linux-arm64/amd64") env = dict(os.environ, GOOS=system, GOARCH=arch, CGO_ENABLED="0") - for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines")): + for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines"), ("pipelines-claude-host", "pipelines-claude-host")): subprocess.run(["go", "build", "-trimpath", "-o", str(binaries / binary), "./cmd/" + command], cwd=ROOT, env=env, check=True) (binaries / "platform.json").write_text(json.dumps({"os": "Darwin" if system == "darwin" else "Linux", "arch": "arm64" if arch == "arm64" and system == "darwin" else "aarch64" if arch == "arm64" else "x86_64"})) diff --git a/scripts/setup-native.py b/scripts/setup-native.py index 609da80e..ba29b0f7 100644 --- a/scripts/setup-native.py +++ b/scripts/setup-native.py @@ -149,6 +149,10 @@ class Setup: content = (f"# {SERVICE_MARKER}\n[Unit]\nDescription=Pipelines native services\n" "\n[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command)) + "\nRestart=on-failure\nRestartSec=5\nTimeoutStopSec=90\nUMask=0077\n" + # The supervisor stops its own children. Only it is + # signalled, so a Claude Code chat host (its own + # session) outlives a restart as it does under launchd. + "KillMode=process\n" "\n[Install]\nWantedBy=default.target\n").encode() path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) self._write_atomic(path, content, 0o600) @@ -444,7 +448,9 @@ class Setup: # --- build and skill ------------------------------------------------- - COMMANDS = ("pipelines-server", "pipelines") + # pipelines-claude-host keeps Claude Code chats alive across server + # restarts; the server finds it next to its own executable. + COMMANDS = ("pipelines-server", "pipelines", "pipelines-claude-host") def build(self, prefix): """Builds the server and CLI into bin/.next; install_binaries moves diff --git a/scripts/test_setup_native.py b/scripts/test_setup_native.py index 63cab2ea..d65aa9e2 100644 --- a/scripts/test_setup_native.py +++ b/scripts/test_setup_native.py @@ -305,7 +305,7 @@ class SetupNativeTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "Node.js 18"): installer.configure_web(Path(directory) / "prefix", setup_native.arguments([])) - def test_build_installs_only_server_and_cli_with_trimpath(self): + def test_build_installs_only_server_cli_and_claude_host_with_trimpath(self): with tempfile.TemporaryDirectory() as directory: recorder = Recorder() installer = self.installer(directory, recorder) @@ -313,11 +313,12 @@ class SetupNativeTests(unittest.TestCase): installer.prepare_layout(prefix) installer.build(prefix) builds = [call for call in recorder.calls if call[:2] == ["go", "build"]] - self.assertEqual(len(builds), 2) + self.assertEqual(len(builds), 3) for call in builds: self.assertIn("-trimpath", call) self.assertTrue(any(call[-1] == "./cmd/pipelines-server" for call in builds)) self.assertTrue(any(call[-1] == "./cmd/pipelines" for call in builds)) + self.assertTrue(any(call[-1] == "./cmd/pipelines-claude-host" for call in builds)) self.assertFalse(any("pipelines-tui" in part for call in recorder.calls for part in call)) def test_layout_creates_private_prefix_and_copies_manage(self):