Merge main; use the shared chatShutdownDetacher for OpenCode and Codex
OpenCode and Codex chats implement Detach (called by chatManager.Close) instead of a parallel DetachSessions interface. Docs follow the Claude Code host and the systemd KillMode=process unit. The opt-in live restart check copies only the stage provider's login row into its scratch OpenCode database, builds the CLI for skill install and removes its scratch tree.
This commit is contained in:
@@ -17,6 +17,7 @@ ARG PIPELINES_COMMIT=unknown
|
||||
ARG PIPELINES_DIRTY=unknown
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags "-X git.a-wan.net/alan/pipelines/internal/buildinfo.Version=${PIPELINES_VERSION} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Commit=${PIPELINES_COMMIT} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Dirty=${PIPELINES_DIRTY}" -o /out/pipelines-server ./cmd/pipelines-server
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags "-X git.a-wan.net/alan/pipelines/internal/buildinfo.Version=${PIPELINES_VERSION} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Commit=${PIPELINES_COMMIT} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Dirty=${PIPELINES_DIRTY}" -o /out/pipelines ./cmd/pipelines
|
||||
RUN CGO_ENABLED=0 go build -trimpath -o /out/pipelines-claude-host ./cmd/pipelines-claude-host
|
||||
|
||||
FROM ${OPENCODE_IMAGE}@${OPENCODE_DIGEST}
|
||||
ARG OPENCODE_VERSION
|
||||
@@ -42,6 +43,7 @@ RUN apk add --no-cache ca-certificates tzdata git curl nodejs npm python3 \
|
||||
&& chown -R pipelines:pipelines /data /workspace /home/pipelines
|
||||
COPY --from=build /out/pipelines-server /usr/local/bin/pipelines-server
|
||||
COPY --from=build /out/pipelines /usr/local/bin/pipelines
|
||||
COPY --from=build /out/pipelines-claude-host /usr/local/bin/pipelines-claude-host
|
||||
COPY --from=build /usr/local/go /usr/local/go
|
||||
COPY --chown=10001:10001 . /workspace
|
||||
ENV HOME=/home/pipelines \
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
// Command pipelines-claude-host keeps one Claude Code CLI process alive across
|
||||
// Pipelines server restarts. The server starts it; it is not run by hand. See
|
||||
// internal/claudehost and docs/chat-backends.md.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(claudehost.Main(os.Args[1:]))
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/buildinfo"
|
||||
"git.a-wan.net/alan/pipelines/internal/claude"
|
||||
"git.a-wan.net/alan/pipelines/internal/discovery"
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
"git.a-wan.net/alan/pipelines/internal/paths"
|
||||
@@ -148,14 +149,15 @@ func main() {
|
||||
options.OpenCodeStatus = openCodeStatus
|
||||
// One manager routes each chat to its owning harness. OpenCode remains the
|
||||
// default; Codex is attached when its app-server control socket is present.
|
||||
// Claude Code chats run in detachable hosts scoped to this database, so a
|
||||
// restart does not end them (internal/claudehost).
|
||||
claude.ConfigureHosts(claude.DefaultHostDir(*db))
|
||||
chats := server.NewChatManager(ctx, options.Chat, store)
|
||||
if len(chats.Backends()) > 0 {
|
||||
options.Chat = chats
|
||||
if detach {
|
||||
defer chats.DetachSessions()
|
||||
} else {
|
||||
defer chats.Close()
|
||||
}
|
||||
// Close detaches a backend whose sessions survive a restart
|
||||
// (chatShutdownDetacher) instead of stopping it.
|
||||
defer chats.Close()
|
||||
}
|
||||
srv := &http.Server{Addr: *addr, Handler: server.HandlerWithOptions(ctx, options, store), ReadHeaderTimeout: 5 * time.Second}
|
||||
// Deferred before WaitRuns so it runs after the runs settle: the owned stage
|
||||
|
||||
+72
-7
@@ -25,8 +25,9 @@ the text the harness recorded), and queued prompts are delivered in order. The
|
||||
older start-up probe of recently used chats still runs for anything not
|
||||
recorded, for example after a crash.
|
||||
|
||||
Per backend (the contract is `chatSessionDetacher` in
|
||||
`internal/server/chat_detach.go`):
|
||||
Per backend (the contract is `chatShutdownDetacher.Detach`, which
|
||||
`chatManager.Close` calls instead of `Close`, plus `chatSessionReleaser` for
|
||||
chat_close; see `internal/server/chat_manager.go` and `chat_detach.go`):
|
||||
|
||||
- **OpenCode**: sessions live in the OpenCode service. An attached endpoint
|
||||
(the operator's CLI-managed service) already outlives the server. Without
|
||||
@@ -39,8 +40,8 @@ Per backend (the contract is `chatSessionDetacher` in
|
||||
control connection, and the next server reconciles through `thread/read`.
|
||||
An approval or question the App Server sent to the old connection is bound to
|
||||
it, so one still pending at shutdown cannot be answered after the restart.
|
||||
- **Claude Code**: closed as before until its host implements
|
||||
`DetachSessions`.
|
||||
- **Claude Code**: its detachable hosts keep the CLI running (see the Claude
|
||||
Code section).
|
||||
|
||||
`PIPELINES_SHUTDOWN_SESSIONS=stop` restores the old shutdown, which closes the
|
||||
chat clients and records nothing. See [Installation](install.md#restarts-keep-sessions-running).
|
||||
@@ -82,9 +83,73 @@ credential or alter global settings. `PIPELINES_CLAUDE` may name the executable;
|
||||
otherwise `claude` is resolved from `PATH` and common per-user locations.
|
||||
|
||||
A new session pins a generated UUID with `--session-id`. When the process is
|
||||
released (idle or server restart), the next prompt resumes the same conversation
|
||||
with `--resume <id>`, so context and the selected model survive. Chat creation is
|
||||
local: the CLI starts on the first prompt or model-catalog read.
|
||||
released (idle, an explicit close, or a host that crashed), the next prompt
|
||||
resumes the same conversation with `--resume <id>`, so context and the selected
|
||||
model survive. Chat creation is local: the CLI starts on the first prompt or
|
||||
model-catalog read.
|
||||
|
||||
### Surviving a server restart
|
||||
|
||||
A server restart does not stop a Claude Code chat. The installed server runs each
|
||||
CLI inside `pipelines-claude-host` (`cmd/pipelines-claude-host`,
|
||||
`internal/claudehost`), a small helper installed next to `pipelines-server`. The
|
||||
host starts in its own session and process group, so neither launchd nor the
|
||||
supervisor stops it with the server (the systemd unit uses `KillMode=process`
|
||||
for the same reason). The host owns the CLI's stdin/stdout. It appends every
|
||||
output line to a per-session spool file with byte offsets and serves a Unix
|
||||
socket. The socket is mode 0600 in a 0700 directory:
|
||||
`<user cache dir>/pipelines/claude-hosts/<hash of the database path>/`. Two
|
||||
servers of one user never adopt each other's chats. Each host also writes a
|
||||
registry file (`<key>.json`) there.
|
||||
|
||||
The socket speaks one JSON object per line: `attach` (deliver output from an
|
||||
offset; a newer attach replaces the older connection), `in` (one stdin frame),
|
||||
`ack` (output processed up to an offset), `status` and `close`. Interrupts,
|
||||
model and mode switches travel as the CLI's own control requests inside `in`. The
|
||||
host snoops these frames to keep the CLI's initialize response (models, account),
|
||||
the applied model and permission mode, a turn's running state, and the control
|
||||
requests nobody has answered yet (`can_use_tool`, `AskUserQuestion`).
|
||||
|
||||
On shutdown the server only disconnects (`Detach`). On start it reads the
|
||||
registry and reattaches to every live host before serving. It replays the spool
|
||||
from its start without broadcasting, which rebuilds the session's stream state,
|
||||
background commands and agents, and pending approvals and questions. Requests
|
||||
already answered are skipped. Output after the last acknowledged offset, written
|
||||
while no server was attached, is then processed live. The turn is shown running
|
||||
if it was running at that offset, watchers get a resync, and the start-up turn
|
||||
probe keeps observing it for notifications. An approval or question asked before
|
||||
the restart can be answered after it. Stop works as usual, and the CLI is never
|
||||
restarted or initialized twice.
|
||||
|
||||
The host stops the CLI (EOF, then SIGINT, then SIGKILL to its process group) and
|
||||
removes its files only when one of these happens:
|
||||
|
||||
- The operator closes the chat (`chat_close`).
|
||||
- The server's idle release or live-process cap fires.
|
||||
- The CLI exits; the server then gets its stderr.
|
||||
- With no server attached, the CLI has been idle for the same 20 minutes, or
|
||||
no server has attached for 24 hours.
|
||||
|
||||
If the host itself dies, the server first tries to reconnect once. If that fails,
|
||||
it stops the orphaned CLI's process group (only when that pid's command line
|
||||
still names the session) and removes the host files. A running turn ends with
|
||||
the notice "the Claude Code host stopped unexpectedly; the next prompt resumes
|
||||
this session with --resume". The next prompt then resumes the session.
|
||||
|
||||
Limits:
|
||||
|
||||
- A container restart (Docker) still ends every host. So does a reboot or
|
||||
logout. The transcript survives and `--resume` applies.
|
||||
- The spool keeps at most 64 MiB of acknowledged output. Older output is
|
||||
dropped, so background-task state from before that point is not rebuilt after
|
||||
a restart. Unacknowledged output is dropped only beyond 256 MiB; the server
|
||||
then logs the gap and clients reload the transcript.
|
||||
- Turn bookkeeping that lived only in server memory restarts fresh: the turn id,
|
||||
and which mid-turn messages are still outstanding. Output that begins a
|
||||
follow-up turn after a reconnect is adopted as a running turn.
|
||||
- Without the host binary (`PIPELINES_CLAUDE_HOST=off`, or not found next to the
|
||||
server or on `PATH`), the CLI is the server's child as before and a restart
|
||||
resumes it.
|
||||
|
||||
Text and reasoning stream from the CLI's partial-message events; completed
|
||||
messages are de-duplicated against the streamed text. Tool calls appear as normal
|
||||
|
||||
+8
-7
@@ -42,7 +42,8 @@ Linux needs `systemctl --user`. Other platforms are refused.
|
||||
2. Builds the Flutter web app in Docker (pinned to `app/.flutter-version`).
|
||||
Builds are cached in `~/.cache/pipelines/app-builds` by a digest of the app
|
||||
sources, so an unchanged app is reused.
|
||||
3. Builds `pipelines-server` and `pipelines` with `go build -trimpath`.
|
||||
3. Builds `pipelines-server`, `pipelines` and `pipelines-claude-host` (keeps
|
||||
Claude Code chats alive across server restarts) with `go build -trimpath`.
|
||||
4. Installs into the prefix: binaries in `bin/`, web app in `lib/app-web`, web
|
||||
proxy in `lib/web`, `lib/manage.py`, and an empty `lib/app-updates` for APKs.
|
||||
5. Runs `pipelines skill install` (the authoring skill for OpenCode and Claude
|
||||
@@ -222,11 +223,11 @@ endpoint, keeps running and is reused. Stop it by killing the `pid` in
|
||||
clients as before, add `"PIPELINES_SHUTDOWN_SESSIONS": "stop"` to
|
||||
`opencode_env` in `<prefix>/pipelines-install.json` and restart.
|
||||
|
||||
Limit on Linux: systemd stops every process in the service's cgroup. The
|
||||
OpenCode services the server starts (stage service, owned chat service) end
|
||||
with `pipelines stop`/`restart`, and their runs fall back as described above.
|
||||
An attached OpenCode endpoint and the Codex App Server are separate and
|
||||
survive. macOS launchd does not kill them.
|
||||
The OpenCode services the server starts (stage service, owned chat service)
|
||||
run in their own process session, so macOS launchd leaves them running. The
|
||||
systemd unit uses `KillMode=process`, so Linux stops only the supervisor, which
|
||||
stops its own children; the detached services survive. An attached OpenCode
|
||||
endpoint and the Codex App Server are separate processes anyway.
|
||||
|
||||
The login service is `~/Library/LaunchAgents/net.a-wan.pipelines.plist` on
|
||||
macOS and `~/.config/systemd/user/net.a-wan.pipelines.service` on Linux. It
|
||||
@@ -238,7 +239,7 @@ autostart, not boot-without-login.
|
||||
| Path | Contents |
|
||||
| --- | --- |
|
||||
| `<prefix>/pipelines-install.json` | manifest (ports, paths, install args, environment) |
|
||||
| `<prefix>/bin/` | `pipelines`, `pipelines-server`, `ntfy` |
|
||||
| `<prefix>/bin/` | `pipelines`, `pipelines-server`, `pipelines-claude-host`, `ntfy` |
|
||||
| `<prefix>/data/pipelines.db` | SQLite database (definitions, runs, chats, settings) |
|
||||
| `<prefix>/data/artifacts/` | run artifacts and session exports |
|
||||
| `<prefix>/data/ntfy/` | ntfy config, cache and accounts |
|
||||
|
||||
@@ -325,6 +325,7 @@ func (s *session) ensureProcess(ctx context.Context) (*process, error) {
|
||||
p := s.proc
|
||||
s.lastUsed = time.Now()
|
||||
s.mu.Unlock()
|
||||
s.awaitReady(ctx)
|
||||
return p, nil
|
||||
}
|
||||
if s.starting != nil {
|
||||
@@ -408,6 +409,7 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
proc := s.proc
|
||||
running := s.running
|
||||
@@ -489,6 +491,7 @@ func (c *Chat) Permissions(ctx context.Context, id string) ([]protocol.ChatPermi
|
||||
if s == nil {
|
||||
return []protocol.ChatPermissionFrame{}, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
frames := make([]protocol.ChatPermissionFrame, 0, len(s.approvals))
|
||||
@@ -505,6 +508,7 @@ func (c *Chat) Forms(ctx context.Context, id string) ([]protocol.ChatForm, error
|
||||
if s == nil {
|
||||
return []protocol.ChatForm{}, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
forms := make([]protocol.ChatForm, 0, len(s.forms))
|
||||
@@ -629,6 +633,7 @@ func (c *Chat) Shells(ctx context.Context, id string) ([]protocol.ChatShell, err
|
||||
if s == nil {
|
||||
return []protocol.ChatShell{}, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
result := []protocol.ChatShell{}
|
||||
|
||||
@@ -4,9 +4,11 @@
|
||||
// mode: stream-json input carries user turns, stream-json output carries
|
||||
// assistant/tool events, and control requests carry interactive prompts
|
||||
// (can_use_tool) and session queries. This adapter owns one CLI process per
|
||||
// active session and mirrors that protocol into Pipelines chat frames. It never
|
||||
// changes the operator's Claude configuration, starts a daemon, or invents a
|
||||
// transcript: history is read back from the CLI's own session files.
|
||||
// active session and mirrors that protocol into Pipelines chat frames. In the
|
||||
// installed server each CLI runs inside a per-session pipelines-claude-host so
|
||||
// it survives a server restart (hosted.go). It never changes the operator's
|
||||
// Claude configuration or invents a transcript: history is read back from the
|
||||
// CLI's own session files.
|
||||
package claude
|
||||
|
||||
import (
|
||||
@@ -14,6 +16,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -22,6 +25,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/workenv"
|
||||
)
|
||||
@@ -96,6 +100,9 @@ type Chat struct {
|
||||
auth authEnv
|
||||
// dirEnv supplies each chat directory's own environment (direnv, mise).
|
||||
dirEnv *workenv.Resolver
|
||||
// hostDir is the private directory of detachable CLI hosts, "" when the
|
||||
// CLI is this server's child (see hosted.go).
|
||||
hostDir string
|
||||
// hinted records sessions whose registry directory has been resolved.
|
||||
hinted sync.Map
|
||||
|
||||
@@ -115,13 +122,30 @@ func New(ctx context.Context) *Chat {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
c := &Chat{ctx: ctx, cancel: cancel, binary: BinaryPath(), dirEnv: workenv.Default, sessions: map[string]*session{}}
|
||||
c := newChat(ctx)
|
||||
if c.hostDir != "" {
|
||||
if err := claudehost.EnsureDir(c.hostDir); err != nil {
|
||||
log.Printf("claude chat: host directory %s: %v; Claude Code chats stop on a server restart", c.hostDir, err)
|
||||
c.hostDir = ""
|
||||
}
|
||||
}
|
||||
c.reattachHosts()
|
||||
go c.reapIdle()
|
||||
return c
|
||||
}
|
||||
|
||||
// Close stops every owned CLI process. Session transcripts stay on disk.
|
||||
// newChat is New without the reattach pass and the reaper (tests).
|
||||
func newChat(ctx context.Context) *Chat {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
c := &Chat{ctx: ctx, cancel: cancel, binary: BinaryPath(), dirEnv: workenv.Default, sessions: map[string]*session{}}
|
||||
hostConfig.Lock()
|
||||
c.hostDir = hostConfig.dir
|
||||
hostConfig.Unlock()
|
||||
return c
|
||||
}
|
||||
|
||||
// Close stops every owned CLI process, hosted ones included. Session
|
||||
// transcripts stay on disk. A server shutdown uses Detach instead.
|
||||
func (c *Chat) Close() {
|
||||
c.cancel()
|
||||
c.mu.Lock()
|
||||
@@ -321,6 +345,10 @@ type session struct {
|
||||
// injectedSeen holds the identities of agent reports and notices already
|
||||
// published live, so a delivery the stream repeats is shown once.
|
||||
injectedSeen map[string]bool
|
||||
|
||||
// silent is set while a reattached host replays output an earlier server
|
||||
// already published: state is rebuilt, nothing is broadcast.
|
||||
silent bool
|
||||
}
|
||||
|
||||
type promptRecord struct {
|
||||
@@ -355,6 +383,10 @@ func (s *session) emit(frame protocol.ChatEventFrame) {
|
||||
s.compaction = nil
|
||||
}
|
||||
}
|
||||
if s.silent {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
targets := make([]chan protocol.ChatEventFrame, 0, len(s.subs))
|
||||
for ch := range s.subs {
|
||||
targets = append(targets, ch)
|
||||
|
||||
@@ -0,0 +1,591 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// Detachable hosts. With hosting configured, a session's CLI is owned by a
|
||||
// pipelines-claude-host process (internal/claudehost) instead of this server,
|
||||
// so a server restart does not end the conversation: the next server finds
|
||||
// the host's registry file, reconnects, rebuilds the session's state from the
|
||||
// spooled output and continues from its last acknowledged offset. Without
|
||||
// hosting (tests, a missing host binary) the CLI is this server's child, as
|
||||
// before, and a restart resumes it with --resume.
|
||||
|
||||
const hostAttachTimeout = 3 * time.Second
|
||||
|
||||
var hostConfig struct {
|
||||
sync.Mutex
|
||||
dir string
|
||||
binary string
|
||||
args []string
|
||||
}
|
||||
|
||||
// ConfigureHosts makes Chats created afterwards run their CLIs in detachable
|
||||
// hosts kept in dir. An empty dir turns hosting off. PIPELINES_CLAUDE_HOST
|
||||
// names the host executable ("off" disables hosting); otherwise
|
||||
// pipelines-claude-host is looked up next to this executable, then on PATH.
|
||||
func ConfigureHosts(dir string) {
|
||||
hostConfig.Lock()
|
||||
defer hostConfig.Unlock()
|
||||
hostConfig.dir = strings.TrimSpace(dir)
|
||||
hostConfig.binary = ""
|
||||
hostConfig.args = nil
|
||||
if hostConfig.dir == "" {
|
||||
return
|
||||
}
|
||||
binary := hostBinaryPath()
|
||||
if binary == "" {
|
||||
log.Printf("claude chat: pipelines-claude-host was not found; Claude Code chats stop on a server restart")
|
||||
hostConfig.dir = ""
|
||||
return
|
||||
}
|
||||
hostConfig.binary = binary
|
||||
}
|
||||
|
||||
// DefaultHostDir is the private host directory for one server instance,
|
||||
// scoped by its database path so two servers of one user never adopt each
|
||||
// other's chats.
|
||||
func DefaultHostDir(scope string) string {
|
||||
base, err := os.UserCacheDir()
|
||||
if err != nil || base == "" {
|
||||
return ""
|
||||
}
|
||||
if absolute, err := filepath.Abs(scope); err == nil {
|
||||
scope = absolute
|
||||
}
|
||||
sum := sha256.Sum256([]byte(scope))
|
||||
return filepath.Join(base, "pipelines", "claude-hosts", hex.EncodeToString(sum[:6]))
|
||||
}
|
||||
|
||||
func hostBinaryPath() string {
|
||||
if value := strings.TrimSpace(os.Getenv("PIPELINES_CLAUDE_HOST")); value != "" {
|
||||
if value == "off" {
|
||||
return ""
|
||||
}
|
||||
return value
|
||||
}
|
||||
if executable, err := os.Executable(); err == nil {
|
||||
candidate := filepath.Join(filepath.Dir(executable), "pipelines-claude-host")
|
||||
if info, err := os.Stat(candidate); err == nil && !info.IsDir() {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
if path, err := exec.LookPath("pipelines-claude-host"); err == nil {
|
||||
return path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hostLink is a process's connection to its host.
|
||||
type hostLink struct {
|
||||
dir string
|
||||
record claudehost.Record
|
||||
|
||||
mu sync.Mutex
|
||||
conn *claudehost.Conn
|
||||
committed int64
|
||||
|
||||
detached atomic.Bool
|
||||
closing atomic.Bool
|
||||
}
|
||||
|
||||
func (l *hostLink) current() *claudehost.Conn {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.conn
|
||||
}
|
||||
|
||||
func (l *hostLink) send(frame claudehost.Frame) error {
|
||||
conn := l.current()
|
||||
if conn == nil {
|
||||
return errProcessClosed
|
||||
}
|
||||
if err := conn.Send(frame); err != nil {
|
||||
return fmt.Errorf("write to the Claude Code host: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replayPlan says how a freshly attached reader treats output it has seen
|
||||
// before: lines below silentUntil were processed by an earlier server and only
|
||||
// rebuild state; control requests below catchUpEnd are applied only when the
|
||||
// host still has them pending (answered ones must not reappear).
|
||||
type replayPlan struct {
|
||||
base int64
|
||||
silentUntil int64
|
||||
catchUpEnd int64
|
||||
pending map[string]bool
|
||||
busyAtAcked bool
|
||||
restore bool
|
||||
}
|
||||
|
||||
// startHosted starts a host for args and attaches to it.
|
||||
func (s *session) startHosted(ctx context.Context, args []string, mode string, env []string) (*process, error) {
|
||||
c := s.chat
|
||||
hostConfig.Lock()
|
||||
binary, hostArgs := hostConfig.binary, append([]string(nil), hostConfig.args...)
|
||||
hostConfig.Unlock()
|
||||
cmd, err := claudehost.Spawn(ctx, claudehost.SpawnOptions{
|
||||
HostBinary: binary, HostArgs: hostArgs, Dir: c.hostDir, SessionID: s.id,
|
||||
Claude: c.binary, Args: args, Env: env, Cwd: s.directory, Idle: idleProcessExpiry,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Reap the host if it ends while this server is still its parent.
|
||||
go func() { _ = cmd.Wait() }()
|
||||
paths := claudehost.PathsFor(c.hostDir, s.id)
|
||||
conn, status, err := claudehost.Attach(paths.Socket, 0, hostAttachTimeout)
|
||||
if err != nil {
|
||||
claudehost.KillStale(c.hostDir, claudehost.Record{SessionID: s.id, HostPID: cmd.Process.Pid})
|
||||
return nil, fmt.Errorf("attach to the Claude Code host: %w", err)
|
||||
}
|
||||
link := &hostLink{dir: c.hostDir, conn: conn, record: claudehost.Record{
|
||||
SessionID: s.id, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: paths.Socket,
|
||||
}}
|
||||
p := newProcess(s, mode)
|
||||
p.link = link
|
||||
p.markReady()
|
||||
go p.readHost(conn, replayPlan{})
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// connectHost attaches to a host found in the registry and prepares a
|
||||
// process for s without registering it. A host that is unreachable, belongs
|
||||
// to another session or never finished initializing is stopped and its files
|
||||
// removed; nil is returned and the next prompt starts a fresh CLI with
|
||||
// --resume.
|
||||
func (c *Chat) connectHost(record claudehost.Record) (*process, *claudehost.Conn, replayPlan) {
|
||||
conn, status, err := claudehost.Attach(record.Socket, 0, hostAttachTimeout)
|
||||
if err != nil {
|
||||
log.Printf("claude chat %s: host unreachable (%v); the next prompt resumes the session", record.SessionID, err)
|
||||
claudehost.KillStale(c.hostDir, record)
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
var models []protocol.ChatModel
|
||||
var account accountIdentity
|
||||
if len(status.Init) > 0 {
|
||||
models, account, err = parseInitialize(status.Init)
|
||||
} else {
|
||||
err = errors.New("the host's CLI never finished initializing")
|
||||
}
|
||||
if status.SessionID != record.SessionID || err != nil {
|
||||
_ = conn.Close()
|
||||
claudehost.KillStale(c.hostDir, record)
|
||||
return nil, nil, replayPlan{}
|
||||
}
|
||||
directory := status.Directory
|
||||
if directory == "" {
|
||||
directory = record.Directory
|
||||
}
|
||||
s := c.sessionFor(record.SessionID, directory)
|
||||
p := newProcess(s, status.Mode)
|
||||
p.link = &hostLink{dir: c.hostDir, conn: conn, committed: status.Acked, record: claudehost.Record{
|
||||
SessionID: record.SessionID, HostPID: status.HostPID, ClaudePID: status.ClaudePID, Socket: record.Socket,
|
||||
}}
|
||||
p.stderr.WriteString(status.Stderr)
|
||||
p.models, p.account, p.appliedModel = models, account, status.Model
|
||||
plan := replayPlan{base: status.Base, silentUntil: status.Acked, catchUpEnd: status.End, pending: map[string]bool{}, busyAtAcked: status.BusyAtAcked, restore: true}
|
||||
for _, pending := range status.Pending {
|
||||
plan.pending[pending.ID] = true
|
||||
}
|
||||
if status.Acked < status.Base {
|
||||
// Output the previous server never processed was dropped by the spool
|
||||
// bound; watchers reload the transcript.
|
||||
log.Printf("claude chat %s: %d bytes of unread output exceeded the host spool", record.SessionID, status.Base-status.Acked)
|
||||
plan.silentUntil = status.Base
|
||||
}
|
||||
return p, conn, plan
|
||||
}
|
||||
|
||||
// adoptLocked applies an attached host's model and account to the session.
|
||||
// Callers hold s.mu.
|
||||
func (s *session) adoptLocked(p *process) {
|
||||
s.models, s.account = p.models, p.account
|
||||
s.appliedModel = p.appliedModel
|
||||
if s.model == "" {
|
||||
s.model = p.appliedModel
|
||||
}
|
||||
s.lastUsed = time.Now()
|
||||
}
|
||||
|
||||
// attachHost adopts a live host found in the registry at server start.
|
||||
func (c *Chat) attachHost(record claudehost.Record) bool {
|
||||
p, conn, plan := c.connectHost(record)
|
||||
if p == nil {
|
||||
return false
|
||||
}
|
||||
s := p.session
|
||||
s.mu.Lock()
|
||||
if s.proc != nil || s.starting != nil {
|
||||
s.mu.Unlock()
|
||||
_ = conn.Close()
|
||||
return false
|
||||
}
|
||||
s.proc = p
|
||||
s.adoptLocked(p)
|
||||
s.mu.Unlock()
|
||||
go p.readHost(conn, plan)
|
||||
return true
|
||||
}
|
||||
|
||||
// reattachHosts adopts every live host of this server's directory. The
|
||||
// registration is synchronous, so a prompt never races a second host for the
|
||||
// same session; the replay that rebuilds each session runs in the background
|
||||
// (see process.ready).
|
||||
func (c *Chat) reattachHosts() {
|
||||
if c.hostDir == "" {
|
||||
return
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, record := range claudehost.Records(c.hostDir) {
|
||||
wg.Add(1)
|
||||
go func(record claudehost.Record) {
|
||||
defer wg.Done()
|
||||
if c.attachHost(record) {
|
||||
log.Printf("claude chat %s: reattached to its running Claude Code host", record.SessionID)
|
||||
}
|
||||
}(record)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// existingHost adopts a host left for this session (one that started after
|
||||
// this server's reattach pass) instead of starting a second one. The caller
|
||||
// holds the session's starting slot and registers the process.
|
||||
func (s *session) existingHost() *process {
|
||||
c := s.chat
|
||||
if c.hostDir == "" {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(claudehost.PathsFor(c.hostDir, s.id).Record)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var record claudehost.Record
|
||||
if json.Unmarshal(data, &record) != nil || record.SessionID != s.id {
|
||||
claudehost.Remove(c.hostDir, s.id)
|
||||
return nil
|
||||
}
|
||||
p, conn, plan := c.connectHost(record)
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.adoptLocked(p)
|
||||
s.mu.Unlock()
|
||||
go p.readHost(conn, plan)
|
||||
return p
|
||||
}
|
||||
|
||||
// readHost is the hosted counterpart of readStdout + wait.
|
||||
func (p *process) readHost(conn *claudehost.Conn, plan replayPlan) {
|
||||
s := p.session
|
||||
live := false
|
||||
goLive := func() {
|
||||
if live {
|
||||
return
|
||||
}
|
||||
live = true
|
||||
s.mu.Lock()
|
||||
s.silent = false
|
||||
turn := ""
|
||||
if plan.restore {
|
||||
if plan.busyAtAcked && !s.running {
|
||||
turn = fmt.Sprintf("claude-%d", time.Now().UnixNano())
|
||||
s.turnID = turn
|
||||
s.running = true
|
||||
s.steered = 0
|
||||
} else if !plan.busyAtAcked {
|
||||
// Output that starts a turn after the reconnect (a follow-up of
|
||||
// messages written before it) is adopted as a running turn.
|
||||
s.awaitFollowUp = true
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if turn != "" {
|
||||
s.emit(protocol.ChatEventFrame{SessionID: s.id, NativeTurnID: turn, Kind: protocol.ChatEventStep, Step: &protocol.ChatStep{Status: protocol.ChatStepStarted, TurnID: turn}})
|
||||
}
|
||||
if plan.restore {
|
||||
// Watchers already attached reload the transcript.
|
||||
s.emit(protocol.ChatEventFrame{SessionID: s.id, Kind: protocol.ChatEventResync})
|
||||
}
|
||||
}
|
||||
if plan.silentUntil > plan.base {
|
||||
s.mu.Lock()
|
||||
s.silent = true
|
||||
s.mu.Unlock()
|
||||
} else {
|
||||
goLive()
|
||||
}
|
||||
if plan.catchUpEnd <= plan.base {
|
||||
p.markReady()
|
||||
}
|
||||
sinceAck := 0
|
||||
for {
|
||||
frame, err := conn.Recv()
|
||||
if err != nil {
|
||||
goLive()
|
||||
p.markReady()
|
||||
if next := p.hostLost(conn, err); next != nil {
|
||||
conn = next
|
||||
plan = replayPlan{}
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
switch frame.Op {
|
||||
case claudehost.OpOut:
|
||||
if frame.Offset >= plan.silentUntil {
|
||||
goLive()
|
||||
}
|
||||
if !(frame.Offset < plan.catchUpEnd && answeredRequest(frame.Data, plan.pending)) {
|
||||
s.handleLine(p, frame.Data)
|
||||
}
|
||||
p.link.mu.Lock()
|
||||
p.link.committed = frame.End
|
||||
p.link.mu.Unlock()
|
||||
sinceAck++
|
||||
if sinceAck >= 64 || !conn.Buffered() {
|
||||
_ = conn.Send(claudehost.Frame{Op: claudehost.OpAck, Offset: frame.End})
|
||||
sinceAck = 0
|
||||
}
|
||||
if frame.End >= plan.silentUntil {
|
||||
goLive()
|
||||
}
|
||||
if frame.End >= plan.catchUpEnd {
|
||||
p.markReady()
|
||||
}
|
||||
case claudehost.OpExit:
|
||||
goLive()
|
||||
p.markReady()
|
||||
if frame.Status != nil && frame.Status.Stderr != "" {
|
||||
p.mu.Lock()
|
||||
if p.stderr.Len() == 0 {
|
||||
p.stderr.WriteString(frame.Status.Stderr)
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
var exitErr error
|
||||
if frame.Error != "" {
|
||||
exitErr = errors.New(frame.Error)
|
||||
}
|
||||
p.link.closing.Store(true)
|
||||
_ = conn.Close()
|
||||
p.finish(exitErr)
|
||||
return
|
||||
case claudehost.OpClosed:
|
||||
p.link.closing.Store(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// answeredRequest reports a replayed CLI control request that is no longer
|
||||
// pending, so an approval already answered is not shown again.
|
||||
func answeredRequest(line []byte, pending map[string]bool) bool {
|
||||
var envelope struct {
|
||||
Type string `json:"type"`
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
if json.Unmarshal(line, &envelope) != nil || envelope.Type != "control_request" {
|
||||
return false
|
||||
}
|
||||
return !pending[envelope.RequestID]
|
||||
}
|
||||
|
||||
func (p *process) markReady() {
|
||||
p.readyOnce.Do(func() { close(p.ready) })
|
||||
}
|
||||
|
||||
// hostLost handles an ended host connection. A deliberate detach or close just
|
||||
// ends the reader; an unexpected loss is retried once (the host may only have
|
||||
// dropped the connection), and otherwise the host is treated as crashed: what
|
||||
// it left is stopped and the turn is settled with a notice, and the next prompt
|
||||
// resumes the session with --resume as after any release.
|
||||
func (p *process) hostLost(conn *claudehost.Conn, cause error) *claudehost.Conn {
|
||||
link := p.link
|
||||
_ = conn.Close()
|
||||
if link.detached.Load() {
|
||||
p.finishDetached()
|
||||
return nil
|
||||
}
|
||||
if link.closing.Load() {
|
||||
p.finish(nil)
|
||||
return nil
|
||||
}
|
||||
link.mu.Lock()
|
||||
committed := link.committed
|
||||
link.mu.Unlock()
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if link.detached.Load() || link.closing.Load() {
|
||||
break
|
||||
}
|
||||
next, _, err := claudehost.Attach(link.record.Socket, committed, hostAttachTimeout)
|
||||
if err == nil {
|
||||
link.mu.Lock()
|
||||
link.conn = next
|
||||
link.mu.Unlock()
|
||||
return next
|
||||
}
|
||||
}
|
||||
if link.detached.Load() {
|
||||
p.finishDetached()
|
||||
return nil
|
||||
}
|
||||
if link.closing.Load() {
|
||||
p.finish(nil)
|
||||
return nil
|
||||
}
|
||||
log.Printf("claude chat %s: host connection lost (%v); falling back to --resume", p.session.id, cause)
|
||||
claudehost.KillStale(link.dir, link.record)
|
||||
p.mu.Lock()
|
||||
p.stderr.Reset()
|
||||
p.mu.Unlock()
|
||||
p.finish(errors.New("the Claude Code host stopped unexpectedly; the next prompt resumes this session with --resume"))
|
||||
return nil
|
||||
}
|
||||
|
||||
// finishDetached ends a process this server let go of on shutdown. The CLI
|
||||
// keeps running in its host; nothing is reported to watchers.
|
||||
func (p *process) finishDetached() {
|
||||
p.mu.Lock()
|
||||
if p.exited {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.exited = true
|
||||
pending := p.pending
|
||||
p.pending = map[string]chan controlReply{}
|
||||
close(p.done)
|
||||
p.mu.Unlock()
|
||||
for _, ch := range pending {
|
||||
select {
|
||||
case ch <- controlReply{err: "the server is shutting down"}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// closeHosted stops the host and its CLI (an explicit release).
|
||||
func (p *process) closeHosted() {
|
||||
link := p.link
|
||||
link.closing.Store(true)
|
||||
if err := link.send(claudehost.Frame{Op: claudehost.OpClose}); err != nil {
|
||||
claudehost.KillStale(link.dir, link.record)
|
||||
}
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(8 * time.Second):
|
||||
claudehost.KillStale(link.dir, link.record)
|
||||
if conn := link.current(); conn != nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// detach lets the host keep running while this server stops.
|
||||
func (p *process) detach() {
|
||||
link := p.link
|
||||
if link == nil {
|
||||
p.close()
|
||||
return
|
||||
}
|
||||
link.detached.Store(true)
|
||||
if conn := link.current(); conn != nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// Detach disconnects from every hosted CLI without stopping it, for a server
|
||||
// shutdown: the next server reattaches. CLIs this server owns directly (no
|
||||
// host) are stopped as Close does.
|
||||
func (c *Chat) Detach() {
|
||||
c.cancel()
|
||||
c.mu.Lock()
|
||||
c.closed = true
|
||||
sessions := make([]*session, 0, len(c.sessions))
|
||||
for _, s := range c.sessions {
|
||||
sessions = append(sessions, s)
|
||||
}
|
||||
c.sessions = map[string]*session{}
|
||||
c.mu.Unlock()
|
||||
var wg sync.WaitGroup
|
||||
for _, s := range sessions {
|
||||
s.mu.Lock()
|
||||
p := s.proc
|
||||
s.proc = nil
|
||||
s.mu.Unlock()
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
p.detach()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// ReleaseSession stops a session's CLI (and its host) for an explicit operator
|
||||
// close. The transcript stays; a later prompt resumes it.
|
||||
func (c *Chat) ReleaseSession(_ context.Context, id string) error {
|
||||
if s := c.session(id); s != nil {
|
||||
s.closeProcess()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProbeTurn reports whether a turn is in flight, after a reattached host's
|
||||
// replay has rebuilt the session, so a turn that survived a restart is
|
||||
// observed and notifies.
|
||||
func (c *Chat) ProbeTurn(ctx context.Context, id string) (bool, error) {
|
||||
s := c.session(id)
|
||||
if s == nil {
|
||||
return false, nil
|
||||
}
|
||||
s.awaitReady(ctx)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.running, nil
|
||||
}
|
||||
|
||||
// awaitReady waits, bounded, until a reattached process has replayed what it
|
||||
// missed, so approvals and the running turn are known before they are read.
|
||||
func (s *session) awaitReady(ctx context.Context) {
|
||||
s.mu.Lock()
|
||||
p := s.proc
|
||||
s.mu.Unlock()
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(10 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-p.ready:
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package claude
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// The test binary doubles as pipelines-claude-host: a host spawned by a test
|
||||
// re-executes it with this variable set.
|
||||
const hostSelfTest = "PIPELINES_CLAUDE_HOST_SELFTEST"
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv(hostSelfTest) == "1" {
|
||||
os.Exit(claudehost.Main(os.Args[1:]))
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
type hostedFixture struct {
|
||||
hostDir string
|
||||
gate string
|
||||
logPath string
|
||||
}
|
||||
|
||||
// hostedSetup configures the fake CLI and detachable hosts in a short private
|
||||
// directory (Unix socket paths are length-limited).
|
||||
func hostedSetup(t *testing.T) hostedFixture {
|
||||
t.Helper()
|
||||
base, err := os.MkdirTemp("/tmp", "pch")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, err := os.ReadFile(filepath.Join("testdata", "fake-claude.py"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
script := filepath.Join(base, "claude")
|
||||
if err := os.WriteFile(script, source, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := hostedFixture{hostDir: filepath.Join(base, "h"), gate: filepath.Join(base, "gate"), logPath: filepath.Join(base, "args.log")}
|
||||
t.Setenv("PIPELINES_CLAUDE", script)
|
||||
t.Setenv("CLAUDE_CONFIG_DIR", filepath.Join(base, "config"))
|
||||
t.Setenv("FAKE_CLAUDE_LOG", fixture.logPath)
|
||||
t.Setenv("FAKE_CLAUDE_GATE", fixture.gate)
|
||||
t.Setenv("PIPELINES_CLAUDE_HOST", executable)
|
||||
t.Setenv(hostSelfTest, "1")
|
||||
ConfigureHosts(fixture.hostDir)
|
||||
t.Cleanup(func() {
|
||||
ConfigureHosts("")
|
||||
for _, record := range claudehost.Records(fixture.hostDir) {
|
||||
claudehost.KillStale(fixture.hostDir, record)
|
||||
}
|
||||
_ = os.RemoveAll(base)
|
||||
})
|
||||
return fixture
|
||||
}
|
||||
|
||||
func hostedChat(t *testing.T) *Chat {
|
||||
t.Helper()
|
||||
chat := New(context.Background())
|
||||
if chat.hostDir == "" {
|
||||
t.Fatal("hosting is not configured")
|
||||
}
|
||||
t.Cleanup(chat.Close)
|
||||
return chat
|
||||
}
|
||||
|
||||
func onlyRecord(t *testing.T, fixture hostedFixture) claudehost.Record {
|
||||
t.Helper()
|
||||
var records []claudehost.Record
|
||||
eventually(t, "a host registry record", func() bool {
|
||||
records = claudehost.Records(fixture.hostDir)
|
||||
return len(records) == 1
|
||||
})
|
||||
return records[0]
|
||||
}
|
||||
|
||||
func alive(pid int) bool { return pid > 0 && syscall.Kill(pid, 0) == nil }
|
||||
|
||||
func textFrame(contains string) func(protocol.ChatEventFrame) bool {
|
||||
return func(frame protocol.ChatEventFrame) bool {
|
||||
return strings.Contains(frame.Delta, contains) || (frame.Part != nil && strings.Contains(frame.Part.Text, contains))
|
||||
}
|
||||
}
|
||||
|
||||
func stepStarted(frame protocol.ChatEventFrame) bool {
|
||||
return frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepStarted
|
||||
}
|
||||
|
||||
// restart stands in for a server restart: the old Chat detaches (its host
|
||||
// keeps running) and a new one is built over the same host directory.
|
||||
func restart(t *testing.T, old *Chat) *Chat {
|
||||
t.Helper()
|
||||
old.Detach()
|
||||
return hostedChat(t)
|
||||
}
|
||||
|
||||
func TestHostedTurnSurvivesRestartAndReplaysMissedOutput(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "gate please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "text before the restart", textFrame("Working"))
|
||||
record := onlyRecord(t, fixture)
|
||||
first.Detach()
|
||||
if !alive(record.HostPID) || !alive(record.ClaudePID) {
|
||||
t.Fatal("detaching must leave the host and its CLI running")
|
||||
}
|
||||
|
||||
// The CLI finishes its turn while no server is attached.
|
||||
if err := os.WriteFile(fixture.gate, nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the turn result in the spool", func() bool {
|
||||
data, _ := os.ReadFile(record.Spool)
|
||||
return strings.Contains(string(data), `"type":"result"`)
|
||||
})
|
||||
|
||||
second := newChat(context.Background())
|
||||
t.Cleanup(second.Close)
|
||||
second.sessionFor(session.ID, session.Directory)
|
||||
frames = subscribe(t, second, session.ID)
|
||||
second.reattachHosts()
|
||||
waitFrame(t, frames, "the restored running turn", stepStarted)
|
||||
waitFrame(t, frames, "text produced while disconnected", textFrame("after the restart"))
|
||||
waitStepEnded(t, frames)
|
||||
if second.TurnRunning(session.ID) {
|
||||
t.Fatal("the replayed result must settle the turn")
|
||||
}
|
||||
if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 {
|
||||
t.Fatalf("the CLI must not be restarted, spawns: %+v", spawns)
|
||||
}
|
||||
// The same CLI keeps serving turns.
|
||||
if err := second.Prompt(context.Background(), session.ID, "hello again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the next turn's text", textFrame("Hello"))
|
||||
waitStepEnded(t, frames)
|
||||
}
|
||||
|
||||
func TestPendingApprovalIsAnsweredAfterRestart(t *testing.T) {
|
||||
hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
if err := first.Prompt(context.Background(), session.ID, "permission please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var before []protocol.ChatPermissionFrame
|
||||
eventually(t, "the approval before the restart", func() bool {
|
||||
before, _ = first.Permissions(context.Background(), session.ID)
|
||||
return len(before) == 1
|
||||
})
|
||||
second := restart(t, first)
|
||||
running, err := second.ProbeTurn(context.Background(), session.ID)
|
||||
if err != nil || !running {
|
||||
t.Fatalf("the restored turn must be running: %v %v", running, err)
|
||||
}
|
||||
after, err := second.Permissions(context.Background(), session.ID)
|
||||
if err != nil || len(after) != 1 || after[0].RequestID != before[0].RequestID {
|
||||
t.Fatalf("the pending approval must survive the restart: %+v %v", after, err)
|
||||
}
|
||||
frames := subscribe(t, second, session.ID)
|
||||
if err := second.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: after[0].RequestID, Decision: protocol.ChatDecisionOnce}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the approved tool result", func(frame protocol.ChatEventFrame) bool {
|
||||
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
|
||||
frame.Part.State.Status == protocol.ChatToolCompleted && len(frame.Part.State.Content) == 1 &&
|
||||
strings.Contains(frame.Part.State.Content[0].Text, "File created")
|
||||
})
|
||||
waitStepEnded(t, frames)
|
||||
|
||||
// An answered approval is not shown again by a later restart's replay.
|
||||
third := restart(t, second)
|
||||
if pending, _ := third.Permissions(context.Background(), session.ID); len(pending) != 0 {
|
||||
t.Fatalf("an answered approval reappeared: %+v", pending)
|
||||
}
|
||||
if third.TurnRunning(session.ID) {
|
||||
t.Fatal("a finished turn must not be restored as running")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuestionSurvivesRestart(t *testing.T) {
|
||||
hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
if err := first.Prompt(context.Background(), session.ID, "question please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the question before the restart", func() bool {
|
||||
forms, _ := first.Forms(context.Background(), session.ID)
|
||||
return len(forms) == 1
|
||||
})
|
||||
second := restart(t, first)
|
||||
forms, err := second.Forms(context.Background(), session.ID)
|
||||
if err != nil || len(forms) != 1 {
|
||||
t.Fatalf("the question must survive the restart: %+v %v", forms, err)
|
||||
}
|
||||
frames := subscribe(t, second, session.ID)
|
||||
if err := second.Answer(context.Background(), protocol.ChatAnswerRequest{SessionID: session.ID, FormID: forms[0].ID, Answers: map[string]any{"q0": "Beta"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the answered question", func(frame protocol.ChatEventFrame) bool {
|
||||
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
|
||||
len(frame.Part.State.Content) == 1 && strings.Contains(frame.Part.State.Content[0].Text, "Beta")
|
||||
})
|
||||
waitStepEnded(t, frames)
|
||||
}
|
||||
|
||||
func TestInterruptAfterRestart(t *testing.T) {
|
||||
hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "long task"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the long turn's text", textFrame("Hello"))
|
||||
second := restart(t, first)
|
||||
if running, _ := second.ProbeTurn(context.Background(), session.ID); !running {
|
||||
t.Fatal("the long turn must still be running after the restart")
|
||||
}
|
||||
frames = subscribe(t, second, session.ID)
|
||||
if err := second.Interrupt(context.Background(), session.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
if second.TurnRunning(session.ID) {
|
||||
t.Fatal("the interrupted turn must settle")
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelAndBackgroundTaskSurviveRestart(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
first := hostedChat(t)
|
||||
session := newCreatedChat(t, first)
|
||||
if _, err := first.Models(context.Background(), session.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := first.SwitchModel(context.Background(), session.ID, Backend, "haiku"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frames := subscribe(t, first, session.ID)
|
||||
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
if shells, _ := first.Shells(context.Background(), session.ID); len(shells) != 1 {
|
||||
t.Fatalf("the background command must be running: %+v", shells)
|
||||
}
|
||||
second := restart(t, first)
|
||||
shells, err := second.Shells(context.Background(), session.ID)
|
||||
if err != nil || len(shells) != 1 || shells[0].Command != "sleep 600" {
|
||||
t.Fatalf("the background command must still be listed after the restart: %+v %v", shells, err)
|
||||
}
|
||||
s := second.session(session.ID)
|
||||
s.mu.Lock()
|
||||
applied, models := s.appliedModel, len(s.models)
|
||||
s.mu.Unlock()
|
||||
if applied != "haiku" || models != 3 {
|
||||
t.Fatalf("the live model and catalog must be restored: %q %d", applied, models)
|
||||
}
|
||||
frames = subscribe(t, second, session.ID)
|
||||
if err := second.PromptWithMessageIDModel(context.Background(), session.ID, "notify me", "", "haiku"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
if shells, _ := second.Shells(context.Background(), session.ID); len(shells) != 0 {
|
||||
t.Fatalf("the notification must finish the restored command: %+v", shells)
|
||||
}
|
||||
if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 {
|
||||
t.Fatalf("the CLI must not be restarted, spawns: %d", len(spawns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostCrashFallsBackToResume(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
chat := hostedChat(t)
|
||||
session := newCreatedChat(t, chat)
|
||||
frames := subscribe(t, chat, session.ID)
|
||||
if err := chat.Prompt(context.Background(), session.ID, "long task"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the long turn's text", textFrame("Hello"))
|
||||
record := onlyRecord(t, fixture)
|
||||
if err := syscall.Kill(record.HostPID, syscall.SIGKILL); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFrame(t, frames, "the host crash notice", func(frame protocol.ChatEventFrame) bool {
|
||||
return frame.Part != nil && frame.Part.State != nil && frame.Part.State.Error != nil &&
|
||||
strings.Contains(frame.Part.State.Error.Message, "--resume")
|
||||
})
|
||||
waitStepEnded(t, frames)
|
||||
eventually(t, "the orphaned CLI to stop", func() bool { return !alive(record.ClaudePID) || isZombie(record.ClaudePID) })
|
||||
if _, err := os.Stat(record.Socket); !os.IsNotExist(err) {
|
||||
t.Fatalf("the crashed host's files must be removed: %v", err)
|
||||
}
|
||||
if err := chat.Prompt(context.Background(), session.ID, "hello again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
spawns := readArgs(t, fixture.logPath)
|
||||
if len(spawns) != 2 || argValue(spawns[1].Args, "--resume") != session.ID {
|
||||
t.Fatalf("the next prompt must resume the session: %+v", spawns)
|
||||
}
|
||||
}
|
||||
|
||||
func isZombie(pid int) bool {
|
||||
var status syscall.WaitStatus
|
||||
got, _ := syscall.Wait4(pid, &status, syscall.WNOHANG, nil)
|
||||
return got == pid
|
||||
}
|
||||
|
||||
func TestReleasingAHostedChatStopsItsHost(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
chat := hostedChat(t)
|
||||
session := newCreatedChat(t, chat)
|
||||
frames := subscribe(t, chat, session.ID)
|
||||
if err := chat.Prompt(context.Background(), session.ID, "hello"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitStepEnded(t, frames)
|
||||
record := onlyRecord(t, fixture)
|
||||
if err := chat.ReleaseSession(context.Background(), session.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the host to exit", func() bool { return !alive(record.HostPID) })
|
||||
eventually(t, "the CLI to exit", func() bool { return !alive(record.ClaudePID) || isZombie(record.ClaudePID) })
|
||||
if records := claudehost.Records(fixture.hostDir); len(records) != 0 {
|
||||
t.Fatalf("a released host must remove its registry record: %+v", records)
|
||||
}
|
||||
// A restarted server finds nothing to reattach; the next prompt resumes.
|
||||
next := hostedChat(t)
|
||||
if next.session(session.ID) != nil {
|
||||
t.Fatal("a released session must not be reattached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostStatusReportsRegistryAndPrivateSocket(t *testing.T) {
|
||||
fixture := hostedSetup(t)
|
||||
chat := hostedChat(t)
|
||||
session := newCreatedChat(t, chat)
|
||||
if _, err := chat.Models(context.Background(), session.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record := onlyRecord(t, fixture)
|
||||
info, err := os.Stat(record.Socket)
|
||||
if err != nil || info.Mode().Perm()&0o077 != 0 {
|
||||
t.Fatalf("the socket must be private: %v %v", info.Mode(), err)
|
||||
}
|
||||
dir, err := os.Stat(fixture.hostDir)
|
||||
if err != nil || dir.Mode().Perm() != 0o700 {
|
||||
t.Fatalf("the host directory must be private: %v %v", dir.Mode(), err)
|
||||
}
|
||||
conn, err := claudehost.Dial(record.Socket, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err := conn.Send(claudehost.Frame{Op: claudehost.OpStatus}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frame, err := conn.Recv()
|
||||
if err != nil || frame.Status == nil || frame.Status.SessionID != session.ID || len(frame.Status.Init) == 0 {
|
||||
raw, _ := json.Marshal(frame)
|
||||
t.Fatalf("status: %s %v", raw, err)
|
||||
}
|
||||
}
|
||||
+76
-22
@@ -14,6 +14,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
@@ -48,6 +49,23 @@ type process struct {
|
||||
// mode is the permission mode this process runs in, guarded by the
|
||||
// session's mu: set at spawn and after a confirmed set_permission_mode.
|
||||
mode string
|
||||
|
||||
// link is set when a detachable host owns the CLI (hosted.go); cmd and
|
||||
// stdin are then unused.
|
||||
link *hostLink
|
||||
// ready closes once a reattached process has replayed the output it
|
||||
// missed; a process this server started is ready at once.
|
||||
ready chan struct{}
|
||||
readyOnce sync.Once
|
||||
// models, account and appliedModel carry an attached host's
|
||||
// initialize state until the session adopts it.
|
||||
models []protocol.ChatModel
|
||||
account accountIdentity
|
||||
appliedModel string
|
||||
}
|
||||
|
||||
func newProcess(s *session, mode string) *process {
|
||||
return &process{session: s, pending: map[string]chan controlReply{}, done: make(chan struct{}), stderrDone: make(chan struct{}), ready: make(chan struct{}), mode: mode}
|
||||
}
|
||||
|
||||
type controlReply struct {
|
||||
@@ -97,6 +115,7 @@ func sessionDisplayName(title string) string {
|
||||
|
||||
// spawn starts the CLI in the session directory. A session whose transcript
|
||||
// already exists resumes it; a new one pins the id we recorded at create time.
|
||||
// With hosting configured the CLI runs in a detachable host (hosted.go).
|
||||
func (s *session) spawn(ctx context.Context) (*process, error) {
|
||||
if err := s.requireBinary(); err != nil {
|
||||
return nil, err
|
||||
@@ -107,34 +126,24 @@ func (s *session) spawn(ctx context.Context) (*process, error) {
|
||||
if info, err := os.Stat(s.directory); err != nil || !info.IsDir() {
|
||||
return nil, fmt.Errorf("Claude session directory is not available: %s", s.directory)
|
||||
}
|
||||
if p := s.existingHost(); p != nil {
|
||||
return p, nil
|
||||
}
|
||||
args, mode := s.cliArgs(findTranscriptFile(s.id) != "")
|
||||
cmd := exec.CommandContext(s.chat.ctx, s.chat.binary, args...)
|
||||
cmd.Dir = s.directory
|
||||
env, notices := s.chat.processEnv(ctx, s.directory)
|
||||
cmd.Env = env
|
||||
s.mu.Lock()
|
||||
s.envNotices = notices
|
||||
s.mu.Unlock()
|
||||
cmd.WaitDelay = 5 * time.Second
|
||||
stdin, err := cmd.StdinPipe()
|
||||
var p *process
|
||||
var err error
|
||||
if s.chat.hostDir != "" {
|
||||
p, err = s.startHosted(ctx, args, mode, env)
|
||||
} else {
|
||||
p, err = s.startDirect(args, mode, env)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start Claude Code: %w", err)
|
||||
}
|
||||
p := &process{session: s, cmd: cmd, stdin: stdin, pending: map[string]chan controlReply{}, done: make(chan struct{}), stderrDone: make(chan struct{}), mode: mode}
|
||||
go p.readStderr(stderr)
|
||||
go p.readStdout(stdout)
|
||||
go p.wait()
|
||||
// The CLI holds can_use_tool requests until the host has initialized. Do that
|
||||
// before any user turn so an approval is never silently denied.
|
||||
initCtx, cancel := context.WithTimeout(ctx, initializeTimeout)
|
||||
@@ -152,11 +161,47 @@ func (s *session) spawn(ctx context.Context) (*process, error) {
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// startDirect runs the CLI as this server's own child.
|
||||
func (s *session) startDirect(args []string, mode string, env []string) (*process, error) {
|
||||
cmd := exec.CommandContext(s.chat.ctx, s.chat.binary, args...)
|
||||
cmd.Dir = s.directory
|
||||
cmd.Env = env
|
||||
cmd.WaitDelay = 5 * time.Second
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start Claude Code: %w", err)
|
||||
}
|
||||
p := newProcess(s, mode)
|
||||
p.cmd, p.stdin = cmd, stdin
|
||||
p.markReady()
|
||||
go p.readStderr(stderr)
|
||||
go p.readStdout(stdout)
|
||||
go p.wait()
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (p *process) initialize(ctx context.Context) ([]protocol.ChatModel, accountIdentity, error) {
|
||||
raw, err := p.request(ctx, "initialize", map[string]any{"subtype": "initialize"})
|
||||
if err != nil {
|
||||
return nil, accountIdentity{}, fmt.Errorf("initialize Claude Code: %w%s", err, p.diagnostic())
|
||||
}
|
||||
return parseInitialize(raw)
|
||||
}
|
||||
|
||||
// parseInitialize reads the CLI's initialize response: its model catalog and
|
||||
// signed-in account.
|
||||
func parseInitialize(raw json.RawMessage) ([]protocol.ChatModel, accountIdentity, error) {
|
||||
var reply struct {
|
||||
Models []struct {
|
||||
Value string `json:"value"`
|
||||
@@ -237,7 +282,6 @@ func (p *process) write(value any) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
p.writeMu.Lock()
|
||||
defer p.writeMu.Unlock()
|
||||
p.mu.Lock()
|
||||
@@ -246,6 +290,10 @@ func (p *process) write(value any) error {
|
||||
if exited {
|
||||
return errProcessClosed
|
||||
}
|
||||
if p.link != nil {
|
||||
return p.link.send(claudehost.Frame{Op: claudehost.OpIn, Data: data})
|
||||
}
|
||||
data = append(data, '\n')
|
||||
if _, err := p.stdin.Write(data); err != nil {
|
||||
return fmt.Errorf("write to Claude Code: %w", err)
|
||||
}
|
||||
@@ -314,7 +362,9 @@ func (p *process) finish(err error) {
|
||||
default:
|
||||
}
|
||||
}
|
||||
_ = p.stdin.Close()
|
||||
if p.stdin != nil {
|
||||
_ = p.stdin.Close()
|
||||
}
|
||||
p.session.processExit(p, err)
|
||||
}
|
||||
|
||||
@@ -325,6 +375,10 @@ func (p *process) close() {
|
||||
if exited {
|
||||
return
|
||||
}
|
||||
if p.link != nil {
|
||||
p.closeHosted()
|
||||
return
|
||||
}
|
||||
_ = p.stdin.Close()
|
||||
if p.cmd.Process != nil {
|
||||
_ = p.cmd.Process.Signal(os.Interrupt)
|
||||
|
||||
+49
@@ -72,6 +72,15 @@ def user_message(text):
|
||||
send({"type": "system", "subtype": "init", "session_id": state["session"],
|
||||
"model": "claude-test", "permissionMode": "default", "cwd": os.getcwd()})
|
||||
message_id = "msg_fake_%d" % state["turn"]
|
||||
if "gate" in text:
|
||||
gated(message_id)
|
||||
return
|
||||
if "background" in text:
|
||||
background(message_id)
|
||||
return
|
||||
if "notify" in text:
|
||||
send({"type": "system", "subtype": "task_notification", "task_id": "b1",
|
||||
"tool_use_id": "toolu_bg_fake", "status": "completed", "summary": "Background command completed (exit code 0)"})
|
||||
send({"type": "stream_event", "event": {"type": "message_start", "message": {"id": message_id}}})
|
||||
send({"type": "stream_event", "event": {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}}})
|
||||
for delta in ("Hello ", "world"):
|
||||
@@ -125,6 +134,46 @@ def user_message(text):
|
||||
finish(message_id)
|
||||
|
||||
|
||||
def wait_for(path):
|
||||
import time
|
||||
while not os.path.exists(path):
|
||||
time.sleep(0.02)
|
||||
|
||||
|
||||
def gated(message_id):
|
||||
"""A turn that streams, then waits for FAKE_CLAUDE_GATE to exist before it
|
||||
streams the rest and finishes: the test restarts the server in between."""
|
||||
gate = os.environ["FAKE_CLAUDE_GATE"]
|
||||
send({"type": "stream_event", "event": {"type": "message_start", "message": {"id": message_id}}})
|
||||
send({"type": "stream_event", "event": {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}}})
|
||||
send({"type": "stream_event", "event": {"type": "content_block_delta", "index": 0,
|
||||
"delta": {"type": "text_delta", "text": "Working"}}})
|
||||
wait_for(gate)
|
||||
send({"type": "stream_event", "event": {"type": "content_block_delta", "index": 0,
|
||||
"delta": {"type": "text_delta", "text": " after the restart"}}})
|
||||
send({"type": "stream_event", "event": {"type": "content_block_stop", "index": 0}})
|
||||
send({"type": "stream_event", "event": {"type": "message_stop"}})
|
||||
text = "Working after the restart"
|
||||
append({"type": "assistant", "uuid": "asst-%d" % state["turn"], "timestamp": stamp(),
|
||||
"message": {"id": message_id, "role": "assistant", "model": "claude-test",
|
||||
"content": [{"type": "text", "text": text}]}})
|
||||
send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test",
|
||||
"content": [{"type": "text", "text": text}]}})
|
||||
finish(message_id)
|
||||
|
||||
|
||||
def background(message_id):
|
||||
"""Starts a background Bash command that a later "notify" turn finishes."""
|
||||
tool_id = "toolu_bg_fake"
|
||||
use = {"type": "tool_use", "id": tool_id, "name": "Bash",
|
||||
"input": {"command": "sleep 600", "run_in_background": True}}
|
||||
send({"type": "assistant", "message": {"id": message_id, "role": "assistant", "model": "claude-test", "content": [use]}})
|
||||
block = {"type": "tool_result", "tool_use_id": tool_id,
|
||||
"content": "Command running in background with ID: b1. Output is being written to: /tmp/b1.output"}
|
||||
send({"type": "user", "message": {"role": "user", "content": [block]}})
|
||||
finish(message_id)
|
||||
|
||||
|
||||
def finish(message_id):
|
||||
# Shape of SDKRateLimitEvent in Claude Code 2.1.316 (synthetic numbers).
|
||||
if os.environ.get("FAKE_CLAUDE_RATE_LIMIT"):
|
||||
|
||||
@@ -348,6 +348,9 @@ func (s *session) claimStreamedLocked(kind, full string, local int) (blockState,
|
||||
// broadcastLocked emits while the session lock is already held: the caller is a
|
||||
// stream-delta path that owns state it just updated.
|
||||
func (s *session) broadcastLocked(frame protocol.ChatEventFrame) {
|
||||
if s.silent {
|
||||
return
|
||||
}
|
||||
targets := make([]chan protocol.ChatEventFrame, 0, len(s.subs))
|
||||
for ch := range s.subs {
|
||||
targets = append(targets, ch)
|
||||
@@ -568,6 +571,9 @@ func (s *session) handleResult(envelope lineEnvelope) {
|
||||
turn := s.turnID
|
||||
running := s.running
|
||||
proc := s.proc
|
||||
if s.silent {
|
||||
proc = nil // replayed: the live usage query is made once caught up
|
||||
}
|
||||
s.running = false
|
||||
s.turnID = ""
|
||||
// Messages written mid-turn that the CLI did not fold into this turn run
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package claudehost
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Conn is a server's connection to one host.
|
||||
type Conn struct {
|
||||
conn net.Conn
|
||||
reader *bufio.Reader
|
||||
writeMu sync.Mutex
|
||||
}
|
||||
|
||||
// Dial connects to a host socket.
|
||||
func Dial(socket string, timeout time.Duration) (*Conn, error) {
|
||||
conn, err := net.DialTimeout("unix", socket, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Conn{conn: conn, reader: bufio.NewReaderSize(conn, 256<<10)}, nil
|
||||
}
|
||||
|
||||
// Send writes one frame.
|
||||
func (c *Conn) Send(frame Frame) error {
|
||||
data, err := json.Marshal(frame)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
c.writeMu.Lock()
|
||||
defer c.writeMu.Unlock()
|
||||
_, err = c.conn.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// Recv reads one frame.
|
||||
func (c *Conn) Recv() (Frame, error) {
|
||||
line, err := c.reader.ReadBytes('\n')
|
||||
if err != nil && len(strings.TrimSpace(string(line))) == 0 {
|
||||
return Frame{}, err
|
||||
}
|
||||
var frame Frame
|
||||
if err := json.Unmarshal(line, &frame); err != nil {
|
||||
return Frame{}, fmt.Errorf("invalid host frame: %w", err)
|
||||
}
|
||||
return frame, nil
|
||||
}
|
||||
|
||||
// Buffered reports whether more frames are already read from the socket.
|
||||
func (c *Conn) Buffered() bool { return c.reader.Buffered() > 0 }
|
||||
|
||||
func (c *Conn) Close() error { return c.conn.Close() }
|
||||
|
||||
// Attach dials socket and starts delivery from offset. It returns the status
|
||||
// the host replied with; output frames follow on the connection.
|
||||
func Attach(socket string, offset int64, timeout time.Duration) (*Conn, *Status, error) {
|
||||
conn, err := Dial(socket, timeout)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
_ = conn.conn.SetDeadline(time.Now().Add(timeout))
|
||||
if err := conn.Send(Frame{Op: OpAttach, Offset: offset}); err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
frame, err := conn.Recv()
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
_ = conn.conn.SetDeadline(time.Time{})
|
||||
if frame.Op != OpStatus || frame.Status == nil {
|
||||
conn.Close()
|
||||
return nil, nil, fmt.Errorf("unexpected host reply %q", frame.Op)
|
||||
}
|
||||
return conn, frame.Status, nil
|
||||
}
|
||||
|
||||
// SpawnOptions starts one host.
|
||||
type SpawnOptions struct {
|
||||
// HostBinary is the pipelines-claude-host executable.
|
||||
HostBinary string
|
||||
// HostArgs are extra arguments placed before the host flags (a test binary
|
||||
// uses them to select host mode).
|
||||
HostArgs []string
|
||||
Dir string
|
||||
SessionID string
|
||||
Claude string
|
||||
Args []string
|
||||
Env []string
|
||||
Cwd string
|
||||
Idle time.Duration
|
||||
}
|
||||
|
||||
// Spawn starts a host in its own session, so it outlives the caller, and
|
||||
// waits until its socket is serving. The caller reaps the returned command.
|
||||
func Spawn(ctx context.Context, options SpawnOptions) (*exec.Cmd, error) {
|
||||
if err := EnsureDir(options.Dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := PathsFor(options.Dir, options.SessionID)
|
||||
args := append([]string(nil), options.HostArgs...)
|
||||
args = append(args, "-dir", options.Dir, "-id", options.SessionID)
|
||||
if options.Idle > 0 {
|
||||
args = append(args, "-idle", options.Idle.String())
|
||||
}
|
||||
args = append(args, "--", options.Claude)
|
||||
args = append(args, options.Args...)
|
||||
cmd := exec.Command(options.HostBinary, args...)
|
||||
cmd.Dir = options.Cwd
|
||||
cmd.Env = options.Env
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
logFile, err := os.OpenFile(paths.Log, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer logFile.Close()
|
||||
cmd.Stderr = logFile
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start the Claude Code host: %w", err)
|
||||
}
|
||||
type ready struct {
|
||||
line string
|
||||
err error
|
||||
}
|
||||
result := make(chan ready, 1)
|
||||
go func() {
|
||||
line, err := bufio.NewReader(stdout).ReadString('\n')
|
||||
result <- ready{line: strings.TrimSpace(line), err: err}
|
||||
}()
|
||||
timer := time.NewTimer(15 * time.Second)
|
||||
defer timer.Stop()
|
||||
fail := func(err error) (*exec.Cmd, error) {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fail(ctx.Err())
|
||||
case <-timer.C:
|
||||
return fail(errors.New("the Claude Code host did not start in time"))
|
||||
case r := <-result:
|
||||
if strings.HasPrefix(r.line, "ready ") {
|
||||
return cmd, nil
|
||||
}
|
||||
message := strings.TrimPrefix(r.line, "error ")
|
||||
if message == "" {
|
||||
message = "the Claude Code host exited during start"
|
||||
if data, err := os.ReadFile(paths.Log); err == nil {
|
||||
if tail := strings.TrimSpace(string(data)); tail != "" {
|
||||
if len(tail) > 400 {
|
||||
tail = tail[len(tail)-400:]
|
||||
}
|
||||
message += ": " + tail
|
||||
}
|
||||
}
|
||||
}
|
||||
return fail(errors.New(message))
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessMatches reports whether pid is alive and its command line mentions
|
||||
// every needle, so a recorded pid that the system reused is never signalled.
|
||||
func ProcessMatches(pid int, needles ...string) bool {
|
||||
if pid <= 1 {
|
||||
return false
|
||||
}
|
||||
if syscall.Kill(pid, 0) != nil {
|
||||
return false
|
||||
}
|
||||
out, err := exec.Command("ps", "-o", "command=", "-p", strconv.Itoa(pid)).Output()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
command := string(out)
|
||||
for _, needle := range needles {
|
||||
if !strings.Contains(command, needle) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// KillStale stops what a dead or unreachable host left behind: the host
|
||||
// itself and its CLI's process group, each only when its command line still
|
||||
// names the session. It then removes the session's host files.
|
||||
func KillStale(dir string, record Record) {
|
||||
if ProcessMatches(record.HostPID, record.SessionID) {
|
||||
_ = syscall.Kill(record.HostPID, syscall.SIGKILL)
|
||||
}
|
||||
if ProcessMatches(record.ClaudePID, record.SessionID) {
|
||||
_ = syscall.Kill(-record.ClaudePID, syscall.SIGTERM)
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) && syscall.Kill(record.ClaudePID, 0) == nil {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
_ = syscall.Kill(-record.ClaudePID, syscall.SIGKILL)
|
||||
}
|
||||
Remove(dir, record.SessionID)
|
||||
}
|
||||
@@ -0,0 +1,710 @@
|
||||
package claudehost
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Defaults for a host started without explicit limits.
|
||||
const (
|
||||
DefaultIdle = 20 * time.Minute
|
||||
DefaultMaxDetached = 24 * time.Hour
|
||||
DefaultSpoolLimit = 64 << 20
|
||||
maxLineBytes = 32 << 20
|
||||
stderrRetained = 8 << 10
|
||||
)
|
||||
|
||||
type lineRef struct{ start, end int64 }
|
||||
|
||||
type transition struct {
|
||||
offset int64
|
||||
busy bool
|
||||
}
|
||||
|
||||
type client struct {
|
||||
conn net.Conn
|
||||
writeMu sync.Mutex
|
||||
gone chan struct{}
|
||||
once sync.Once
|
||||
exitSent bool
|
||||
}
|
||||
|
||||
func (c *client) send(frame Frame) error {
|
||||
data, err := json.Marshal(frame)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
c.writeMu.Lock()
|
||||
defer c.writeMu.Unlock()
|
||||
_, err = c.conn.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *client) close() {
|
||||
c.once.Do(func() {
|
||||
close(c.gone)
|
||||
_ = c.conn.Close()
|
||||
})
|
||||
}
|
||||
|
||||
type host struct {
|
||||
dir string
|
||||
id string
|
||||
paths Paths
|
||||
idle time.Duration
|
||||
maxDetached time.Duration
|
||||
spoolLimit int64
|
||||
listener net.Listener
|
||||
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
stdinMu sync.Mutex
|
||||
|
||||
fileMu sync.RWMutex // guards spool replacement against readers
|
||||
mu sync.Mutex
|
||||
spool *os.File
|
||||
base int64
|
||||
end int64
|
||||
lines []lineRef
|
||||
acked int64
|
||||
trans []transition
|
||||
busy bool
|
||||
// pending maps unanswered CLI control request ids to their offsets.
|
||||
pending map[string]int64
|
||||
initID string
|
||||
init json.RawMessage
|
||||
modelReqs map[string]string
|
||||
modeReqs map[string]string
|
||||
model string
|
||||
mode string
|
||||
exited bool
|
||||
exitErr string
|
||||
exitedAt time.Time
|
||||
stderr strings.Builder
|
||||
changed chan struct{}
|
||||
active *client
|
||||
detached time.Time
|
||||
activity time.Time
|
||||
stopping bool
|
||||
stopped chan struct{}
|
||||
claudeEnd chan struct{}
|
||||
}
|
||||
|
||||
// Main runs a host. args are the host flags, then "--", then the Claude Code
|
||||
// executable and its arguments. The working directory and environment are the
|
||||
// CLI's own. It returns the process exit code.
|
||||
func Main(args []string) int {
|
||||
flags := flag.NewFlagSet("pipelines-claude-host", flag.ContinueOnError)
|
||||
dir := flags.String("dir", "", "private host directory (registry, socket, spool)")
|
||||
id := flags.String("id", "", "Claude session id")
|
||||
idle := flags.Duration("idle", DefaultIdle, "release an idle CLI after this long with no server attached")
|
||||
maxDetached := flags.Duration("max-detached", DefaultMaxDetached, "release the CLI after this long with no server attached, even when busy")
|
||||
spoolLimit := flags.Int64("spool-limit", DefaultSpoolLimit, "bytes of acknowledged output kept in the spool")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
command := flags.Args()
|
||||
if len(command) == 0 || *id == "" || *dir == "" {
|
||||
fmt.Fprintln(os.Stderr, "usage: pipelines-claude-host -dir DIR -id SESSION -- claude ARGS...")
|
||||
return 2
|
||||
}
|
||||
log.SetFlags(log.LstdFlags)
|
||||
log.SetPrefix("claude-host " + *id + ": ")
|
||||
syscall.Umask(0o077)
|
||||
if err := EnsureDir(*dir); err != nil {
|
||||
log.Print(err)
|
||||
return 1
|
||||
}
|
||||
h := &host{
|
||||
dir: *dir, id: *id, paths: PathsFor(*dir, *id), idle: *idle, maxDetached: *maxDetached, spoolLimit: *spoolLimit,
|
||||
pending: map[string]int64{}, modelReqs: map[string]string{}, modeReqs: map[string]string{},
|
||||
changed: make(chan struct{}), stopped: make(chan struct{}), claudeEnd: make(chan struct{}),
|
||||
trans: []transition{{offset: 0, busy: false}}, activity: time.Now(), detached: time.Now(),
|
||||
}
|
||||
h.model, h.mode = flagValue(command, "--model"), flagValue(command, "--permission-mode")
|
||||
if err := h.start(command); err != nil {
|
||||
log.Print(err)
|
||||
fmt.Fprintln(os.Stdout, "error "+strings.ReplaceAll(err.Error(), "\n", " "))
|
||||
return 1
|
||||
}
|
||||
// The server waits for this line, then stops reading our stdout.
|
||||
fmt.Fprintln(os.Stdout, "ready "+h.paths.Socket)
|
||||
if null, err := os.OpenFile(os.DevNull, os.O_RDWR, 0); err == nil {
|
||||
_ = syscall.Dup2(int(null.Fd()), 1)
|
||||
_ = syscall.Dup2(int(null.Fd()), 0)
|
||||
_ = null.Close()
|
||||
}
|
||||
signals := make(chan os.Signal, 2)
|
||||
signal.Ignore(syscall.SIGHUP, syscall.SIGPIPE)
|
||||
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
||||
go func() {
|
||||
<-signals
|
||||
h.shutdown("signal")
|
||||
}()
|
||||
go h.accept()
|
||||
go h.watchIdle()
|
||||
<-h.stopped
|
||||
return 0
|
||||
}
|
||||
|
||||
func flagValue(args []string, name string) string {
|
||||
for index, arg := range args {
|
||||
if arg == name && index+1 < len(args) {
|
||||
return args[index+1]
|
||||
}
|
||||
if strings.HasPrefix(arg, name+"=") {
|
||||
return strings.TrimPrefix(arg, name+"=")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (h *host) start(command []string) error {
|
||||
if conn, err := net.DialTimeout("unix", h.paths.Socket, time.Second); err == nil {
|
||||
_ = conn.Close()
|
||||
return errors.New("a host is already serving this session")
|
||||
}
|
||||
_ = os.Remove(h.paths.Socket)
|
||||
spool, err := os.OpenFile(h.paths.Spool, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.spool = spool
|
||||
listener, err := net.Listen("unix", h.paths.Socket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Chmod(h.paths.Socket, 0o600)
|
||||
h.listener = listener
|
||||
cmd := exec.Command(command[0], command[1:]...)
|
||||
cmd.Env = os.Environ()
|
||||
// Its own process group, so the CLI and its tool subprocesses stop together.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = listener.Close()
|
||||
_ = os.Remove(h.paths.Socket)
|
||||
return fmt.Errorf("start Claude Code: %w", err)
|
||||
}
|
||||
h.cmd, h.stdin = cmd, stdin
|
||||
cwd, _ := os.Getwd()
|
||||
record := Record{Version: Version, SessionID: h.id, HostPID: os.Getpid(), ClaudePID: cmd.Process.Pid,
|
||||
Socket: h.paths.Socket, Spool: h.paths.Spool, Directory: cwd, Created: time.Now().UnixMilli()}
|
||||
data, _ := json.Marshal(record)
|
||||
if err := writeFileAtomic(h.paths.Record, data); err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
return err
|
||||
}
|
||||
stderrDone := make(chan struct{})
|
||||
go h.readStderr(stderr, stderrDone)
|
||||
readDone := make(chan struct{})
|
||||
go h.readStdout(stdout, readDone)
|
||||
go func() {
|
||||
err := cmd.Wait()
|
||||
<-readDone
|
||||
select {
|
||||
case <-stderrDone:
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.exited = true
|
||||
h.exitedAt = time.Now()
|
||||
if err != nil {
|
||||
h.exitErr = err.Error()
|
||||
}
|
||||
h.broadcastLocked()
|
||||
h.mu.Unlock()
|
||||
close(h.claudeEnd)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *host) readStderr(stderr io.Reader, done chan struct{}) {
|
||||
defer close(done)
|
||||
buffer := make([]byte, 4<<10)
|
||||
for {
|
||||
n, err := stderr.Read(buffer)
|
||||
if n > 0 {
|
||||
h.mu.Lock()
|
||||
if remaining := stderrRetained - h.stderr.Len(); remaining > 0 {
|
||||
if n > remaining {
|
||||
n = remaining
|
||||
}
|
||||
h.stderr.Write(buffer[:n])
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *host) readStdout(stdout io.Reader, done chan struct{}) {
|
||||
defer close(done)
|
||||
reader := bufio.NewReaderSize(stdout, 64<<10)
|
||||
for {
|
||||
line, err := reader.ReadBytes('\n')
|
||||
if len(line) > 0 && len(line) <= maxLineBytes {
|
||||
trimmed := strings.TrimSpace(string(line))
|
||||
if trimmed != "" {
|
||||
h.appendLine([]byte(trimmed))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// broadcastLocked wakes every sender. Callers hold h.mu.
|
||||
func (h *host) broadcastLocked() {
|
||||
close(h.changed)
|
||||
h.changed = make(chan struct{})
|
||||
}
|
||||
|
||||
type snoop struct {
|
||||
Type string `json:"type"`
|
||||
Subtype string `json:"subtype"`
|
||||
RequestID string `json:"request_id"`
|
||||
ParentToolUseID string `json:"parent_tool_use_id"`
|
||||
Request struct {
|
||||
Subtype string `json:"subtype"`
|
||||
Model string `json:"model"`
|
||||
Mode string `json:"mode"`
|
||||
} `json:"request"`
|
||||
Response struct {
|
||||
Subtype string `json:"subtype"`
|
||||
RequestID string `json:"request_id"`
|
||||
Response json.RawMessage `json:"response"`
|
||||
} `json:"response"`
|
||||
}
|
||||
|
||||
func (h *host) setBusyLocked(busy bool, offset int64) {
|
||||
if h.busy == busy {
|
||||
return
|
||||
}
|
||||
h.busy = busy
|
||||
h.trans = append(h.trans, transition{offset: offset, busy: busy})
|
||||
}
|
||||
|
||||
func (h *host) busyAtLocked(offset int64) bool {
|
||||
busy := false
|
||||
for _, t := range h.trans {
|
||||
if t.offset > offset {
|
||||
break
|
||||
}
|
||||
busy = t.busy
|
||||
}
|
||||
return busy
|
||||
}
|
||||
|
||||
func (h *host) appendLine(line []byte) {
|
||||
var env snoop
|
||||
_ = json.Unmarshal(line, &env)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
start := h.end
|
||||
data := append(append([]byte(nil), line...), '\n')
|
||||
h.fileMu.RLock()
|
||||
_, err := h.spool.WriteAt(data, start-h.base)
|
||||
h.fileMu.RUnlock()
|
||||
if err != nil {
|
||||
log.Printf("spool write: %v", err)
|
||||
}
|
||||
h.end += int64(len(data))
|
||||
h.lines = append(h.lines, lineRef{start: start, end: h.end})
|
||||
h.activity = time.Now()
|
||||
switch env.Type {
|
||||
case "control_request":
|
||||
if env.RequestID != "" {
|
||||
h.pending[env.RequestID] = start
|
||||
}
|
||||
case "control_cancel_request":
|
||||
delete(h.pending, env.RequestID)
|
||||
case "control_response":
|
||||
id := env.Response.RequestID
|
||||
if env.Response.Subtype == "success" {
|
||||
if id != "" && id == h.initID {
|
||||
h.init = append(json.RawMessage(nil), env.Response.Response...)
|
||||
}
|
||||
if model, ok := h.modelReqs[id]; ok {
|
||||
h.model = model
|
||||
}
|
||||
if mode, ok := h.modeReqs[id]; ok {
|
||||
h.mode = mode
|
||||
}
|
||||
}
|
||||
delete(h.modelReqs, id)
|
||||
delete(h.modeReqs, id)
|
||||
case "result":
|
||||
h.setBusyLocked(false, h.end)
|
||||
// A finished turn no longer waits on any approval.
|
||||
h.pending = map[string]int64{}
|
||||
case "stream_event", "assistant":
|
||||
if env.ParentToolUseID == "" {
|
||||
h.setBusyLocked(true, start)
|
||||
}
|
||||
}
|
||||
h.compactLocked()
|
||||
h.broadcastLocked()
|
||||
}
|
||||
|
||||
// compactLocked bounds the spool. Acknowledged output beyond the limit is
|
||||
// dropped first; unacknowledged output is dropped only beyond four times the
|
||||
// limit (a server gone for very long), which a reconnecting server sees as a
|
||||
// gap because Acked < Base.
|
||||
func (h *host) compactLocked() {
|
||||
size := h.end - h.base
|
||||
if size <= h.spoolLimit {
|
||||
return
|
||||
}
|
||||
keep := h.acked
|
||||
if h.end-keep > 4*h.spoolLimit {
|
||||
keep = h.end - h.spoolLimit
|
||||
}
|
||||
index := sort.Search(len(h.lines), func(i int) bool { return h.lines[i].start >= keep })
|
||||
if index == 0 || index >= len(h.lines) {
|
||||
return
|
||||
}
|
||||
keep = h.lines[index].start
|
||||
if keep-h.base < size/4 {
|
||||
return // not worth rewriting yet
|
||||
}
|
||||
next, err := os.OpenFile(h.paths.Spool+".next", os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
h.fileMu.Lock()
|
||||
defer h.fileMu.Unlock()
|
||||
if _, err := io.Copy(next, io.NewSectionReader(h.spool, keep-h.base, h.end-keep)); err != nil {
|
||||
_ = next.Close()
|
||||
_ = os.Remove(h.paths.Spool + ".next")
|
||||
return
|
||||
}
|
||||
if err := os.Rename(h.paths.Spool+".next", h.paths.Spool); err != nil {
|
||||
_ = next.Close()
|
||||
return
|
||||
}
|
||||
_ = h.spool.Close()
|
||||
h.spool = next
|
||||
h.base = keep
|
||||
h.lines = append([]lineRef(nil), h.lines[index:]...)
|
||||
}
|
||||
|
||||
func (h *host) statusLocked() *Status {
|
||||
pending := make([]Pending, 0, len(h.pending))
|
||||
for id, offset := range h.pending {
|
||||
pending = append(pending, Pending{ID: id, Offset: offset})
|
||||
}
|
||||
sort.Slice(pending, func(i, j int) bool { return pending[i].Offset < pending[j].Offset })
|
||||
cwd, _ := os.Getwd()
|
||||
status := &Status{
|
||||
Version: Version, SessionID: h.id, HostPID: os.Getpid(),
|
||||
Base: h.base, End: h.end, Acked: h.acked, Busy: h.busy, BusyAtAcked: h.busyAtLocked(h.acked),
|
||||
Init: h.init, Model: h.model, Mode: h.mode, Pending: pending,
|
||||
Exited: h.exited, ExitErr: h.exitErr, Stderr: h.stderr.String(), Directory: cwd,
|
||||
}
|
||||
if h.cmd != nil && h.cmd.Process != nil {
|
||||
status.ClaudePID = h.cmd.Process.Pid
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (h *host) accept() {
|
||||
for {
|
||||
conn, err := h.listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c := &client{conn: conn, gone: make(chan struct{})}
|
||||
go h.serveClient(c)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *host) serveClient(c *client) {
|
||||
defer h.dropClient(c)
|
||||
reader := bufio.NewReaderSize(c.conn, 64<<10)
|
||||
for {
|
||||
line, err := reader.ReadBytes('\n')
|
||||
if len(strings.TrimSpace(string(line))) > 0 {
|
||||
var frame Frame
|
||||
if json.Unmarshal(line, &frame) != nil {
|
||||
_ = c.send(Frame{Op: OpError, Error: "invalid frame"})
|
||||
} else if !h.handle(c, frame) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *host) dropClient(c *client) {
|
||||
c.close()
|
||||
h.mu.Lock()
|
||||
if h.active == c {
|
||||
h.active = nil
|
||||
h.detached = time.Now()
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// handle runs one client frame; false ends the connection.
|
||||
func (h *host) handle(c *client, frame Frame) bool {
|
||||
switch frame.Op {
|
||||
case OpAttach:
|
||||
h.mu.Lock()
|
||||
previous := h.active
|
||||
h.active = c
|
||||
status := h.statusLocked()
|
||||
h.mu.Unlock()
|
||||
if previous != nil && previous != c {
|
||||
previous.close()
|
||||
}
|
||||
if err := c.send(Frame{Op: OpStatus, Status: status}); err != nil {
|
||||
return false
|
||||
}
|
||||
go h.deliver(c, frame.Offset)
|
||||
case OpIn:
|
||||
h.writeIn(frame.Data)
|
||||
case OpAck:
|
||||
h.mu.Lock()
|
||||
if frame.Offset > h.acked && frame.Offset <= h.end {
|
||||
h.acked = frame.Offset
|
||||
h.trimTransitionsLocked()
|
||||
}
|
||||
h.mu.Unlock()
|
||||
case OpStatus:
|
||||
h.mu.Lock()
|
||||
status := h.statusLocked()
|
||||
h.mu.Unlock()
|
||||
_ = c.send(Frame{Op: OpStatus, Status: status})
|
||||
case OpClose:
|
||||
h.stopClaude()
|
||||
_ = c.send(Frame{Op: OpClosed})
|
||||
h.shutdown("closed")
|
||||
return false
|
||||
default:
|
||||
_ = c.send(Frame{Op: OpError, Error: "unknown op " + frame.Op})
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *host) trimTransitionsLocked() {
|
||||
keep := 0
|
||||
for i, t := range h.trans {
|
||||
if t.offset <= h.acked {
|
||||
keep = i
|
||||
}
|
||||
}
|
||||
h.trans = append([]transition(nil), h.trans[keep:]...)
|
||||
}
|
||||
|
||||
func (h *host) writeIn(data json.RawMessage) {
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
var env snoop
|
||||
_ = json.Unmarshal(data, &env)
|
||||
h.mu.Lock()
|
||||
h.activity = time.Now()
|
||||
switch env.Type {
|
||||
case "user":
|
||||
h.setBusyLocked(true, h.end)
|
||||
case "control_response":
|
||||
delete(h.pending, env.Response.RequestID)
|
||||
case "control_request":
|
||||
switch env.Request.Subtype {
|
||||
case "initialize":
|
||||
h.initID = env.RequestID
|
||||
case "set_model":
|
||||
h.modelReqs[env.RequestID] = env.Request.Model
|
||||
case "set_permission_mode":
|
||||
h.modeReqs[env.RequestID] = env.Request.Mode
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
line := append(append([]byte(nil), data...), '\n')
|
||||
h.stdinMu.Lock()
|
||||
defer h.stdinMu.Unlock()
|
||||
if _, err := h.stdin.Write(line); err != nil {
|
||||
log.Printf("write to Claude Code: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// deliver streams spooled lines from offset, then the exit, to one client.
|
||||
func (h *host) deliver(c *client, offset int64) {
|
||||
cursor := offset
|
||||
for {
|
||||
h.mu.Lock()
|
||||
if h.active != c {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
index := sort.Search(len(h.lines), func(i int) bool { return h.lines[i].start >= cursor })
|
||||
if index < len(h.lines) {
|
||||
ref := h.lines[index]
|
||||
h.fileMu.RLock()
|
||||
h.mu.Unlock()
|
||||
data := make([]byte, ref.end-ref.start)
|
||||
_, err := h.spool.ReadAt(data, ref.start-h.base)
|
||||
h.fileMu.RUnlock()
|
||||
if err != nil {
|
||||
log.Printf("spool read: %v", err)
|
||||
c.close()
|
||||
return
|
||||
}
|
||||
if err := c.send(Frame{Op: OpOut, Offset: ref.start, End: ref.end, Data: json.RawMessage(strings.TrimSpace(string(data)))}); err != nil {
|
||||
c.close()
|
||||
return
|
||||
}
|
||||
cursor = ref.end
|
||||
continue
|
||||
}
|
||||
if h.exited && !c.exitSent {
|
||||
c.exitSent = true
|
||||
status := h.statusLocked()
|
||||
h.mu.Unlock()
|
||||
_ = c.send(Frame{Op: OpExit, Status: status, Error: status.ExitErr})
|
||||
// The server has everything; it closes the connection, then we go.
|
||||
go func() {
|
||||
select {
|
||||
case <-c.gone:
|
||||
case <-time.After(30 * time.Second):
|
||||
}
|
||||
h.shutdown("exited")
|
||||
}()
|
||||
continue
|
||||
}
|
||||
changed := h.changed
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case <-changed:
|
||||
case <-c.gone:
|
||||
return
|
||||
case <-h.stopped:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stopClaude ends the CLI: stdin EOF, then an interrupt, then a kill, to the
|
||||
// CLI's whole process group.
|
||||
func (h *host) stopClaude() {
|
||||
if h.cmd == nil || h.cmd.Process == nil {
|
||||
return
|
||||
}
|
||||
h.stdinMu.Lock()
|
||||
_ = h.stdin.Close()
|
||||
h.stdinMu.Unlock()
|
||||
pid := h.cmd.Process.Pid
|
||||
select {
|
||||
case <-h.claudeEnd:
|
||||
return
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
_ = syscall.Kill(-pid, syscall.SIGINT)
|
||||
select {
|
||||
case <-h.claudeEnd:
|
||||
return
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
_ = syscall.Kill(-pid, syscall.SIGKILL)
|
||||
select {
|
||||
case <-h.claudeEnd:
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
// shutdown stops the CLI, removes the host files and ends Main.
|
||||
func (h *host) shutdown(reason string) {
|
||||
h.mu.Lock()
|
||||
if h.stopping {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.stopping = true
|
||||
active := h.active
|
||||
h.mu.Unlock()
|
||||
log.Printf("stopping (%s)", reason)
|
||||
h.stopClaude()
|
||||
_ = h.listener.Close()
|
||||
Remove(h.dir, h.id)
|
||||
_ = os.Remove(h.paths.Spool + ".next")
|
||||
if active != nil {
|
||||
active.close()
|
||||
}
|
||||
close(h.stopped)
|
||||
}
|
||||
|
||||
// watchIdle releases the CLI while no server is attached: after the idle
|
||||
// expiry when nothing is in flight (the same release the server applies), and
|
||||
// after maxDetached regardless. A CLI that exited with nobody attached is
|
||||
// cleaned up after the idle expiry.
|
||||
func (h *host) watchIdle() {
|
||||
tick := h.idle / 10
|
||||
if tick > 5*time.Second {
|
||||
tick = 5 * time.Second
|
||||
}
|
||||
if tick < 50*time.Millisecond {
|
||||
tick = 50 * time.Millisecond
|
||||
}
|
||||
ticker := time.NewTicker(tick)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-h.stopped:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
now := time.Now()
|
||||
h.mu.Lock()
|
||||
attached := h.active != nil
|
||||
idle := !h.busy && len(h.pending) == 0 && now.Sub(h.detached) > h.idle && now.Sub(h.activity) > h.idle
|
||||
tooLong := now.Sub(h.detached) > h.maxDetached
|
||||
exitedLong := h.exited && now.Sub(h.exitedAt) > h.idle
|
||||
h.mu.Unlock()
|
||||
if attached {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case exitedLong:
|
||||
h.shutdown("exited with no server attached")
|
||||
return
|
||||
case idle:
|
||||
h.shutdown("idle with no server attached")
|
||||
return
|
||||
case tooLong:
|
||||
h.shutdown("no server attached for " + h.maxDetached.String())
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
// Package claudehost keeps one Claude Code CLI process alive across Pipelines
|
||||
// server restarts.
|
||||
//
|
||||
// The server cannot own a `claude -p` process directly and also restart: the
|
||||
// CLI's stdin/stdout pipes die with their parent. A host is a small helper
|
||||
// process in its own session that owns those pipes instead. It spools the
|
||||
// CLI's stream-json output to a bounded per-session file with byte offsets and
|
||||
// serves one local Unix socket (0600, in a 0700 directory) on which the server
|
||||
// writes stdin frames, reads output from an offset, acknowledges what it has
|
||||
// processed, and finally closes the host. A restarted server finds live hosts
|
||||
// through their registry files, reconnects and replays from its last
|
||||
// acknowledged offset.
|
||||
//
|
||||
// The wire format is one JSON object per line in both directions.
|
||||
package claudehost
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Version is the host protocol version reported in every status.
|
||||
const Version = 1
|
||||
|
||||
// Client-to-host operations.
|
||||
const (
|
||||
// OpAttach starts (or restarts) delivery from Frame.Offset. The host
|
||||
// replies with OpStatus, then streams OpOut frames. A newer attach replaces
|
||||
// an older connection.
|
||||
OpAttach = "attach"
|
||||
// OpIn writes Frame.Data, one JSON line, to the CLI's stdin.
|
||||
OpIn = "in"
|
||||
// OpAck commits that output up to Frame.Offset has been processed.
|
||||
OpAck = "ack"
|
||||
// OpStatus asks for a status snapshot without changing delivery.
|
||||
OpStatus = "status"
|
||||
// OpClose stops the CLI and the host. The host replies OpClosed.
|
||||
OpClose = "close"
|
||||
)
|
||||
|
||||
// Host-to-client operations.
|
||||
const (
|
||||
OpOut = "out"
|
||||
OpExit = "exit"
|
||||
OpClosed = "closed"
|
||||
OpError = "error"
|
||||
)
|
||||
|
||||
// Frame is one line on the socket.
|
||||
type Frame struct {
|
||||
Op string `json:"op"`
|
||||
Offset int64 `json:"offset,omitempty"`
|
||||
End int64 `json:"end,omitempty"`
|
||||
Data json.RawMessage `json:"data,omitempty"`
|
||||
Status *Status `json:"status,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Pending is a CLI control request (can_use_tool, a hook) that nobody has
|
||||
// answered yet. It survives a disconnected server.
|
||||
type Pending struct {
|
||||
ID string `json:"id"`
|
||||
Offset int64 `json:"offset"`
|
||||
}
|
||||
|
||||
// Status is the host's snapshot of its CLI.
|
||||
type Status struct {
|
||||
Version int `json:"version"`
|
||||
SessionID string `json:"session_id"`
|
||||
HostPID int `json:"host_pid"`
|
||||
ClaudePID int `json:"claude_pid"`
|
||||
// Base is the oldest offset still in the spool; End is the offset after the
|
||||
// last spooled line; Acked is the last offset a client committed.
|
||||
Base int64 `json:"base"`
|
||||
End int64 `json:"end"`
|
||||
Acked int64 `json:"acked"`
|
||||
// Busy is whether a turn is in flight now; BusyAtAcked is whether one was
|
||||
// in flight at the Acked offset, which is where a reconnecting server's
|
||||
// live processing resumes.
|
||||
Busy bool `json:"busy"`
|
||||
BusyAtAcked bool `json:"busy_at_acked"`
|
||||
// Init is the CLI's initialize response (models, account), so a
|
||||
// reconnecting server does not initialize the CLI twice.
|
||||
Init json.RawMessage `json:"init,omitempty"`
|
||||
// Model and Mode are the CLI's model and permission mode as last applied
|
||||
// (start flags, then successful set_model / set_permission_mode).
|
||||
Model string `json:"model,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Pending []Pending `json:"pending"`
|
||||
Exited bool `json:"exited,omitempty"`
|
||||
ExitErr string `json:"exit_error,omitempty"`
|
||||
Stderr string `json:"stderr,omitempty"`
|
||||
// Directory is the CLI's working directory.
|
||||
Directory string `json:"directory,omitempty"`
|
||||
}
|
||||
|
||||
// Record is the registry file a host writes next to its socket so a
|
||||
// restarted server can find it.
|
||||
type Record struct {
|
||||
Version int `json:"version"`
|
||||
SessionID string `json:"session_id"`
|
||||
HostPID int `json:"host_pid"`
|
||||
ClaudePID int `json:"claude_pid"`
|
||||
Socket string `json:"socket"`
|
||||
Spool string `json:"spool"`
|
||||
Directory string `json:"directory"`
|
||||
Created int64 `json:"created"`
|
||||
}
|
||||
|
||||
// Key is the short file name used for a session's host files. Unix socket
|
||||
// paths are limited to about 100 bytes, so the session id is hashed.
|
||||
func Key(sessionID string) string {
|
||||
sum := sha256.Sum256([]byte(sessionID))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// Paths are one session's files inside a host directory.
|
||||
type Paths struct {
|
||||
Socket, Record, Spool, Log string
|
||||
}
|
||||
|
||||
func PathsFor(dir, sessionID string) Paths {
|
||||
key := Key(sessionID)
|
||||
return Paths{
|
||||
Socket: filepath.Join(dir, key+".sock"),
|
||||
Record: filepath.Join(dir, key+".json"),
|
||||
Spool: filepath.Join(dir, key+".spool"),
|
||||
Log: filepath.Join(dir, key+".log"),
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDir creates a private host directory.
|
||||
func EnsureDir(dir string) error {
|
||||
if strings.TrimSpace(dir) == "" || !filepath.IsAbs(dir) {
|
||||
return errors.New("the Claude host directory must be absolute")
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Chmod(dir, 0o700)
|
||||
}
|
||||
|
||||
// Records lists the registry files in dir. Unreadable files are skipped.
|
||||
func Records(dir string) []Record {
|
||||
matches, _ := filepath.Glob(filepath.Join(dir, "*.json"))
|
||||
records := make([]Record, 0, len(matches))
|
||||
for _, path := range matches {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var record Record
|
||||
if json.Unmarshal(data, &record) != nil || record.SessionID == "" || record.Socket == "" {
|
||||
continue
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
// Remove deletes one session's host files.
|
||||
func Remove(dir, sessionID string) {
|
||||
paths := PathsFor(dir, sessionID)
|
||||
for _, path := range []string{paths.Record, paths.Socket, paths.Spool, paths.Log} {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
|
||||
func writeFileAtomic(path string, data []byte) error {
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
@@ -176,13 +176,13 @@ func (c *Chat) Close() {
|
||||
_ = c.conn.Close(websocket.StatusNormalClosure, "")
|
||||
}
|
||||
|
||||
// DetachSessions is the server-restart shutdown: threads and their turns live
|
||||
// Detach is the server-restart shutdown (chatShutdownDetacher): threads and their turns live
|
||||
// in the Codex App Server, not in this client, so closing the control
|
||||
// connection leaves them running; no turn is interrupted. The next server
|
||||
// process reconnects and reconciles each thread through thread/read. An
|
||||
// approval or question the App Server sent to this connection is bound to it,
|
||||
// so one still pending at shutdown is not answerable after the restart.
|
||||
func (c *Chat) DetachSessions() { c.Close() }
|
||||
func (c *Chat) Detach() { c.Close() }
|
||||
|
||||
func (c *Chat) readLoop() {
|
||||
for {
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// A server restart detaches from the App Server without interrupting the
|
||||
// thread's turn; the next server's fresh connection still sees the turn in
|
||||
// progress, then completed, through the read-only turn list.
|
||||
func TestDetachSessionsLeavesTurnRunningForNextConnection(t *testing.T) {
|
||||
func TestDetachLeavesTurnRunningForNextConnection(t *testing.T) {
|
||||
var completed atomic.Bool
|
||||
var mu sync.Mutex
|
||||
var methods []string
|
||||
@@ -29,7 +29,7 @@ func TestDetachSessionsLeavesTurnRunningForNextConnection(t *testing.T) {
|
||||
if running, err := before.ProbeTurn(context.Background(), fixtureThread); err != nil || !running {
|
||||
t.Fatalf("turn not running before the restart: %v %v", running, err)
|
||||
}
|
||||
before.DetachSessions()
|
||||
before.Detach()
|
||||
after := modelTestChat(t, appServer)
|
||||
if running, err := after.ProbeTurn(context.Background(), fixtureThread); err != nil || !running {
|
||||
t.Fatalf("turn did not survive the restart: %v %v", running, err)
|
||||
|
||||
@@ -40,7 +40,7 @@ func Uninstall(executable string, args []string, in io.Reader, out io.Writer) er
|
||||
}
|
||||
dir := filepath.Dir(resolved)
|
||||
var files []string
|
||||
for _, command := range []string{"pipelines", "pipelines-server", "pipelines-tui"} {
|
||||
for _, command := range []string{"pipelines", "pipelines-server", "pipelines-tui", "pipelines-claude-host"} {
|
||||
path := filepath.Join(dir, command)
|
||||
info, err := buildinfo.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
|
||||
@@ -304,12 +304,12 @@ func (c *Chat) Interrupt(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DetachSessions is the server-restart shutdown. Sessions, turns, questions,
|
||||
// Detach is the server-restart shutdown (chatShutdownDetacher). Sessions, turns, questions,
|
||||
// permissions and shells live in the OpenCode service, not in this adapter, so
|
||||
// there is nothing to stop: the connection's owner decides whether the service
|
||||
// itself outlives the server (an attached endpoint and the detached owned chat
|
||||
// service do; a V1 serve child does not).
|
||||
func (c *Chat) DetachSessions() {}
|
||||
func (c *Chat) Detach() {}
|
||||
|
||||
// Messages reads one session's stored turns. On the owned serve backend it
|
||||
// reuses Connection.Messages, including its read-only OutputFormat readback
|
||||
|
||||
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
@@ -58,6 +59,13 @@ func handleChatClose(ctx context.Context, request protocol.Message, broker chatB
|
||||
}
|
||||
// A closed chat takes no more turns: prompts it still held are dropped.
|
||||
hub.clearQueue(ctx, sessionID)
|
||||
// A harness that keeps the agent alive across restarts (Claude Code's
|
||||
// host) stops it now; the transcript stays and a later prompt resumes it.
|
||||
if releaser, ok := broker.(chatSessionReleaser); ok {
|
||||
if err := releaser.ReleaseSession(ctx, sessionID); err != nil {
|
||||
log.Printf("chat session %s: release after close: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
session, found, err := registry.OperatorSession(ctx, sessionID)
|
||||
if err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
||||
|
||||
@@ -13,23 +13,20 @@ import (
|
||||
// Chat sessions survive a Pipelines restart. Only an operator action (closing
|
||||
// the chat, or Stop on its turn) stops one.
|
||||
//
|
||||
// Per-backend contract. A chat backend whose sessions can outlive the server
|
||||
// process implements chatSessionDetacher. On shutdown the server calls
|
||||
// DetachSessions instead of Close: the backend releases its own connection or
|
||||
// process handles but must not interrupt, abort or kill any session, turn,
|
||||
// Per-backend contract (chatShutdownDetacher and chatSessionReleaser in
|
||||
// chat_manager.go). On shutdown chatManager.Close calls Detach on a backend
|
||||
// that implements it instead of Close: the backend releases its own connection
|
||||
// or process handles but must not interrupt, abort or kill any session, turn,
|
||||
// question or background task. On the next start the server reattaches through
|
||||
// the ordinary broker surface: chatTurnProber.ProbeTurn (is the turn still
|
||||
// running?), Stream (re-subscribe), Permissions/Forms (pending questions and
|
||||
// approvals), Shells (background tasks) and Messages (read back a turn that
|
||||
// ended while the server was down). A backend without DetachSessions is closed
|
||||
// as before, and its sessions end with the server.
|
||||
// ended while the server was down). chat_close calls ReleaseSession on a
|
||||
// backend that keeps an agent alive until released. A backend without Detach
|
||||
// is closed as before, and its sessions end with the server.
|
||||
//
|
||||
// OpenCode (an attached or detached owned service) and Codex (its App Server)
|
||||
// implement it; a Claude Code host that keeps its CLI processes alive can
|
||||
// implement the same method.
|
||||
type chatSessionDetacher interface {
|
||||
DetachSessions()
|
||||
}
|
||||
// OpenCode (an attached or detached owned service), Codex (its App Server) and
|
||||
// Claude Code (its detachable hosts) implement Detach.
|
||||
|
||||
// chatLiveStore records, on shutdown, the chats left live on their harness so
|
||||
// the next start reattaches exactly those; *pipeline.Store implements it.
|
||||
|
||||
@@ -166,14 +166,32 @@ func (m *chatManager) harness(backend string) chatBroker {
|
||||
return client
|
||||
}
|
||||
|
||||
// chatShutdownDetacher is implemented by a backend whose live sessions can
|
||||
// outlive this server: on shutdown it lets them go instead of stopping them,
|
||||
// and the next server reattaches (Claude Code's detachable hosts).
|
||||
type chatShutdownDetacher interface {
|
||||
Detach()
|
||||
}
|
||||
|
||||
// chatSessionReleaser is implemented by a backend that keeps a session's agent
|
||||
// running until it is explicitly released; chat_close releases it.
|
||||
type chatSessionReleaser interface {
|
||||
ReleaseSession(ctx context.Context, sessionID string) error
|
||||
}
|
||||
|
||||
func closeHarness(client chatBroker) {
|
||||
if detacher, ok := client.(chatShutdownDetacher); ok {
|
||||
detacher.Detach()
|
||||
return
|
||||
}
|
||||
if closer, ok := client.(interface{ Close() }); ok {
|
||||
closer.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// Close releases the owned Codex and Claude Code clients, if any, and stops
|
||||
// later lazy attachment.
|
||||
// Close lets go of the Codex and Claude Code clients, if any, and stops later
|
||||
// lazy attachment. It runs at server shutdown: a backend whose sessions can
|
||||
// survive it (chatShutdownDetacher) is detached, not stopped.
|
||||
func (m *chatManager) Close() {
|
||||
m.mu.Lock()
|
||||
m.closed = true
|
||||
@@ -183,26 +201,17 @@ func (m *chatManager) Close() {
|
||||
closeHarness(claudeClient)
|
||||
}
|
||||
|
||||
// DetachSessions is the shutdown counterpart of Close for a server that keeps
|
||||
// chat sessions running across a restart (see chat_detach.go). Each harness
|
||||
// that implements chatSessionDetacher releases its handles without stopping a
|
||||
// session; one that does not is closed as before. The OpenCode connection's
|
||||
// lifetime belongs to the caller (cmd/pipelines-server), so it is only detached.
|
||||
func (m *chatManager) DetachSessions() {
|
||||
m.mu.Lock()
|
||||
m.closed = true
|
||||
codexClient, claudeClient := m.codex, m.claude
|
||||
m.mu.Unlock()
|
||||
if detacher, ok := m.opencode.(chatSessionDetacher); ok {
|
||||
detacher.DetachSessions()
|
||||
// ReleaseSession releases a session's agent on its own harness after an
|
||||
// explicit close. A harness without that notion is left as is.
|
||||
func (m *chatManager) ReleaseSession(ctx context.Context, sessionID string) error {
|
||||
adapter, err := m.brokerFor(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for _, client := range []chatBroker{codexClient, claudeClient} {
|
||||
if detacher, ok := client.(chatSessionDetacher); ok {
|
||||
detacher.DetachSessions()
|
||||
continue
|
||||
}
|
||||
closeHarness(client)
|
||||
if releaser, ok := adapter.(chatSessionReleaser); ok {
|
||||
return releaser.ReleaseSession(ctx, sessionID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Backends lists the harnesses available for a new session.
|
||||
|
||||
@@ -29,8 +29,8 @@ import (
|
||||
// PIPELINES_RESTART_LIVE=1 go test ./internal/server -run TestRestartKeepsRealOpenCodeSessions -v -timeout 15m
|
||||
//
|
||||
// It builds pipelines-server, runs it on a scratch HOME/XDG tree, a scratch
|
||||
// database and a free loopback port (with only the operator's OpenCode
|
||||
// provider login copied in), starts a chat turn and a one-stage run that each
|
||||
// database and a free loopback port (with only the operator's opencode-go
|
||||
// provider login row copied into the scratch OpenCode database), starts a chat turn and a one-stage run that each
|
||||
// run `sleep 30`, restarts the server mid-turn, and checks both finish
|
||||
// instead of being aborted. Every scratch process is killed afterwards.
|
||||
func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
|
||||
@@ -46,6 +46,9 @@ func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("scratch root %s", root)
|
||||
// Registered first, so it runs last: after the scratch processes are
|
||||
// stopped, remove the tree (it holds a copied provider login row).
|
||||
t.Cleanup(func() { _ = os.RemoveAll(root) })
|
||||
home := filepath.Join(root, "home")
|
||||
env := scratchEnvironment(t, realHome, home)
|
||||
binary := filepath.Join(root, "pipelines-server")
|
||||
@@ -56,8 +59,14 @@ func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
|
||||
}
|
||||
// Chat creation activates the manager skill, so install the current
|
||||
// source's skills into the scratch harness roots.
|
||||
skills := exec.Command("go", "run", "./cmd/pipelines", "skill", "install")
|
||||
skills.Dir, skills.Env = filepath.Join("..", ".."), env
|
||||
cli := filepath.Join(root, "pipelines")
|
||||
buildCLI := exec.Command("go", "build", "-o", cli, "./cmd/pipelines")
|
||||
buildCLI.Dir = filepath.Join("..", "..")
|
||||
if out, err := buildCLI.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build CLI: %v\n%s", err, out)
|
||||
}
|
||||
skills := exec.Command(cli, "skill", "install")
|
||||
skills.Env = env
|
||||
if out, err := skills.CombinedOutput(); err != nil {
|
||||
t.Fatalf("skill install: %v\n%s", err, out)
|
||||
}
|
||||
@@ -81,6 +90,10 @@ func TestRestartKeepsRealOpenCodeSessions(t *testing.T) {
|
||||
server.stop()
|
||||
killScratchServices(t, home)
|
||||
})
|
||||
// OpenCode 2.x keeps provider logins in its own database. The chat service
|
||||
// has just created the scratch one; copy only the stage provider's login
|
||||
// row into it before the stage service starts (scratch, owner-only).
|
||||
copyProviderLogin(t, filepath.Join(realHome, ".local", "share", "opencode", "opencode.db"), filepath.Join(home, ".local", "share", "opencode", "opencode.db"), "opencode-go")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
@@ -163,13 +176,6 @@ func scratchEnvironment(t *testing.T, realHome, home string) []string {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
auth, err := os.ReadFile(filepath.Join(realHome, ".local", "share", "opencode", "auth.json"))
|
||||
if err != nil {
|
||||
t.Skipf("no OpenCode login to copy: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(data, "opencode", "auth.json"), auth, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A cold model catalog can leave the first service unable to resolve the
|
||||
// model; seed the public catalog cache (not a credential).
|
||||
if catalog, err := os.ReadFile(filepath.Join(realHome, ".cache", "opencode", "models.json")); err == nil {
|
||||
@@ -197,6 +203,28 @@ func scratchEnvironment(t *testing.T, realHome, home string) []string {
|
||||
"PIPELINES_OPENCODE="+opencodePath(), "PIPELINES_NTFY_URL=", "PIPELINES_MDNS=off")
|
||||
}
|
||||
|
||||
func copyProviderLogin(t *testing.T, real, scratch, provider string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(real); err != nil {
|
||||
t.Skipf("no OpenCode login database: %v", err)
|
||||
}
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for {
|
||||
if _, err := os.Stat(scratch); err == nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("scratch OpenCode database was not created")
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
script := "ATTACH '" + real + "' AS real; INSERT OR REPLACE INTO credential SELECT * FROM real.credential WHERE integration_id='" + provider + "'; SELECT changes();"
|
||||
out, err := exec.Command("sqlite3", "-bail", scratch, script).CombinedOutput()
|
||||
if err != nil || strings.TrimSpace(string(out)) == "0" {
|
||||
t.Skipf("could not copy the %s login into the scratch OpenCode database: %v %s", provider, err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func opencodePath() string {
|
||||
if path, err := exec.LookPath("opencode"); err == nil {
|
||||
return path
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ def package(output, version, target):
|
||||
if system not in ("darwin", "linux") or arch not in ("arm64", "amd64"):
|
||||
raise ValueError("Supported: source, darwin-arm64/amd64, linux-arm64/amd64")
|
||||
env = dict(os.environ, GOOS=system, GOARCH=arch, CGO_ENABLED="0")
|
||||
for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines")):
|
||||
for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines"), ("pipelines-claude-host", "pipelines-claude-host")):
|
||||
subprocess.run(["go", "build", "-trimpath", "-o", str(binaries / binary), "./cmd/" + command],
|
||||
cwd=ROOT, env=env, check=True)
|
||||
(binaries / "platform.json").write_text(json.dumps({"os": "Darwin" if system == "darwin" else "Linux", "arch": "arm64" if arch == "arm64" and system == "darwin" else "aarch64" if arch == "arm64" else "x86_64"}))
|
||||
|
||||
@@ -149,6 +149,10 @@ class Setup:
|
||||
content = (f"# {SERVICE_MARKER}\n[Unit]\nDescription=Pipelines native services\n"
|
||||
"\n[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command))
|
||||
+ "\nRestart=on-failure\nRestartSec=5\nTimeoutStopSec=90\nUMask=0077\n"
|
||||
# The supervisor stops its own children. Only it is
|
||||
# signalled, so a Claude Code chat host (its own
|
||||
# session) outlives a restart as it does under launchd.
|
||||
"KillMode=process\n"
|
||||
"\n[Install]\nWantedBy=default.target\n").encode()
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
self._write_atomic(path, content, 0o600)
|
||||
@@ -444,7 +448,9 @@ class Setup:
|
||||
|
||||
# --- build and skill -------------------------------------------------
|
||||
|
||||
COMMANDS = ("pipelines-server", "pipelines")
|
||||
# pipelines-claude-host keeps Claude Code chats alive across server
|
||||
# restarts; the server finds it next to its own executable.
|
||||
COMMANDS = ("pipelines-server", "pipelines", "pipelines-claude-host")
|
||||
|
||||
def build(self, prefix):
|
||||
"""Builds the server and CLI into bin/.next; install_binaries moves
|
||||
|
||||
@@ -305,7 +305,7 @@ class SetupNativeTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "Node.js 18"):
|
||||
installer.configure_web(Path(directory) / "prefix", setup_native.arguments([]))
|
||||
|
||||
def test_build_installs_only_server_and_cli_with_trimpath(self):
|
||||
def test_build_installs_only_server_cli_and_claude_host_with_trimpath(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
recorder = Recorder()
|
||||
installer = self.installer(directory, recorder)
|
||||
@@ -313,11 +313,12 @@ class SetupNativeTests(unittest.TestCase):
|
||||
installer.prepare_layout(prefix)
|
||||
installer.build(prefix)
|
||||
builds = [call for call in recorder.calls if call[:2] == ["go", "build"]]
|
||||
self.assertEqual(len(builds), 2)
|
||||
self.assertEqual(len(builds), 3)
|
||||
for call in builds:
|
||||
self.assertIn("-trimpath", call)
|
||||
self.assertTrue(any(call[-1] == "./cmd/pipelines-server" for call in builds))
|
||||
self.assertTrue(any(call[-1] == "./cmd/pipelines" for call in builds))
|
||||
self.assertTrue(any(call[-1] == "./cmd/pipelines-claude-host" for call in builds))
|
||||
self.assertFalse(any("pipelines-tui" in part for call in recorder.calls for part in call))
|
||||
|
||||
def test_layout_creates_private_prefix_and_copies_manage(self):
|
||||
|
||||
Reference in New Issue
Block a user