diff --git a/Dockerfile b/Dockerfile index 969aea6f..17658b3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ ARG PIPELINES_DIRTY=unknown RUN CGO_ENABLED=0 go build -trimpath -ldflags "-X git.a-wan.net/alan/pipelines/internal/buildinfo.Version=${PIPELINES_VERSION} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Commit=${PIPELINES_COMMIT} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Dirty=${PIPELINES_DIRTY}" -o /out/pipelines-server ./cmd/pipelines-server RUN CGO_ENABLED=0 go build -trimpath -ldflags "-X git.a-wan.net/alan/pipelines/internal/buildinfo.Version=${PIPELINES_VERSION} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Commit=${PIPELINES_COMMIT} -X git.a-wan.net/alan/pipelines/internal/buildinfo.Dirty=${PIPELINES_DIRTY}" -o /out/pipelines ./cmd/pipelines RUN CGO_ENABLED=0 go build -trimpath -o /out/pipelines-claude-host ./cmd/pipelines-claude-host +RUN CGO_ENABLED=0 go build -trimpath -o /out/pipelines-codex-host ./cmd/pipelines-codex-host FROM ${OPENCODE_IMAGE}@${OPENCODE_DIGEST} ARG OPENCODE_VERSION @@ -44,6 +45,7 @@ RUN apk add --no-cache ca-certificates tzdata git curl nodejs npm python3 \ COPY --from=build /out/pipelines-server /usr/local/bin/pipelines-server COPY --from=build /out/pipelines /usr/local/bin/pipelines COPY --from=build /out/pipelines-claude-host /usr/local/bin/pipelines-claude-host +COPY --from=build /out/pipelines-codex-host /usr/local/bin/pipelines-codex-host COPY --from=build /usr/local/go /usr/local/go COPY --chown=10001:10001 . /workspace ENV HOME=/home/pipelines \ diff --git a/cmd/pipelines-codex-host/main.go b/cmd/pipelines-codex-host/main.go new file mode 100644 index 00000000..e7e4689b --- /dev/null +++ b/cmd/pipelines-codex-host/main.go @@ -0,0 +1,10 @@ +// Command pipelines-codex-host retains the existing Codex App Server connection +// across Pipelines server restarts. The server starts it automatically. +package main + +import ( + "git.a-wan.net/alan/pipelines/internal/codexhost" + "os" +) + +func main() { os.Exit(codexhost.Main(os.Args[1:])) } diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index 1632e826..48170856 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -14,6 +14,7 @@ import ( "git.a-wan.net/alan/pipelines/internal/buildinfo" "git.a-wan.net/alan/pipelines/internal/claude" + "git.a-wan.net/alan/pipelines/internal/codex" "git.a-wan.net/alan/pipelines/internal/discovery" "git.a-wan.net/alan/pipelines/internal/opencode" "git.a-wan.net/alan/pipelines/internal/paths" @@ -150,8 +151,10 @@ func main() { // One manager routes each chat to its owning harness. OpenCode remains the // default; Codex is attached when its app-server control socket is present. // Claude Code chats run in detachable hosts scoped to this database, so a - // restart does not end them (internal/claudehost). + // restart does not end them (internal/claudehost). Codex hosts retain its + // original App Server connection so pending interactions remain answerable. claude.ConfigureHosts(claude.DefaultHostDir(*db)) + codex.ConfigureHosts(codex.DefaultHostDir(*db)) chats := server.NewChatManager(ctx, options.Chat, store) if len(chats.Backends()) > 0 { options.Chat = chats diff --git a/docs/chat-backends.md b/docs/chat-backends.md index b6e9b369..912354e1 100644 --- a/docs/chat-backends.md +++ b/docs/chat-backends.md @@ -36,10 +36,12 @@ chat_close; see `internal/server/chat_manager.go` and `chat_detach.go`): `/pipelines/private/opencode-chat-service.json`). The next server reuses it at the same address and password, and it is not stopped on shutdown. A V1 serve child still ends with the server. -- **Codex**: threads and turns live in the App Server. Detaching closes the - control connection, and the next server reconciles through `thread/read`. - An approval or question the App Server sent to the old connection is bound to - it, so one still pending at shutdown cannot be answered after the restart. +- **Codex**: threads and turns live in the App Server. The installed server + connects through `pipelines-codex-host`, which retains the original App Server + connection and unanswered approvals/questions while Pipelines detaches. A new + server receives those same callbacks and reconciles ordinary turn state through + readback. Direct mode (hosting disabled or helper absent) still loses pending + connection-bound interactions on restart. - **Claude Code**: its detachable hosts keep the CLI running (see the Claude Code section). @@ -72,6 +74,35 @@ not a fake empty success or silent auto-approval. App Server socket loss does not provide durable callback recovery; a disconnected transport is reported as an error rather than claiming old requests are still actionable. +### Pending interactions across restarts + +The installed `pipelines-codex-host` helper owns one WebSocket connection to the +existing Codex App Server Unix socket. It never starts a second Codex daemon or +changes Codex settings. Helpers are scoped by database path and upstream socket, +with mode-0600 Unix sockets and singleton locks under the mode-0700 directory +`/pipelines/codex-hosts//`. + +On reconnect the helper returns its cached initialization handshake and replays +only unanswered server requests, preserving exact callback IDs and parameters. +Responses and `serverRequest/resolved` remove pending entries. Old/new client RPC +IDs are remapped so a delayed response for an old Pipelines process cannot answer +a new process's call. A new attach closes the obsolete downstream connection. +Ordinary notifications are not spooled; the existing thread readback repairs gaps. + +State is bounded to 128 callbacks / 8 MiB and 256 in-flight client calls, with a +16 MiB frame limit. Exceeding pending bounds closes the bridge without granting, +declining or otherwise auto-answering an interaction. A detached bridge expires +after 20 minutes with no pending interactions, or after 24 hours even if pending. +An upstream App Server restart, a bridge crash, or expiry loses its connection- +bound callbacks: no disk JSON can recreate them on a new connection. The initial +deployment cannot recover requests already owned by the old direct connection. + +`PIPELINES_CODEX_HOST` overrides the helper executable; `off` disables hosting. +The server otherwise finds it beside `pipelines-server`, then on PATH. Without +hosting, direct-mode fixtures and intentional direct connections retain their +previous behaviour and limitation. Installing a new helper binary does not kill +an existing helper or its pending requests. + ## Claude Code The adapter in `internal/claude` owns one Claude Code CLI process per active diff --git a/docs/install.md b/docs/install.md index adda804b..fdf32f03 100644 --- a/docs/install.md +++ b/docs/install.md @@ -42,8 +42,9 @@ Linux needs `systemctl --user`. Other platforms are refused. 2. Builds the Flutter web app in Docker (pinned to `app/.flutter-version`). Builds are cached in `~/.cache/pipelines/app-builds` by a digest of the app sources, so an unchanged app is reused. -3. Builds `pipelines-server`, `pipelines` and `pipelines-claude-host` (keeps - Claude Code chats alive across server restarts) with `go build -trimpath`. +3. Builds `pipelines-server`, `pipelines`, `pipelines-claude-host` and + `pipelines-codex-host` (retain Claude processes and Codex interaction callbacks + across server restarts) with `go build -trimpath`. 4. Installs into the prefix: binaries in `bin/`, web app in `lib/app-web`, web proxy in `lib/web`, `lib/manage.py`, and an empty `lib/app-updates` for APKs. 5. Runs `pipelines skill install` (the authoring skill for OpenCode and Claude @@ -239,7 +240,7 @@ autostart, not boot-without-login. | Path | Contents | | --- | --- | | `/pipelines-install.json` | manifest (ports, paths, install args, environment) | -| `/bin/` | `pipelines`, `pipelines-server`, `pipelines-claude-host`, `ntfy` | +| `/bin/` | `pipelines`, `pipelines-server`, `pipelines-claude-host`, `pipelines-codex-host`, `ntfy` | | `/data/pipelines.db` | SQLite database (definitions, runs, chats, settings) | | `/data/artifacts/` | run artifacts and session exports | | `/data/ntfy/` | ntfy config, cache and accounts | diff --git a/docs/notes/codex-pending-interactions-restart.md b/docs/notes/codex-pending-interactions-restart.md new file mode 100644 index 00000000..53c90e93 --- /dev/null +++ b/docs/notes/codex-pending-interactions-restart.md @@ -0,0 +1,49 @@ +# Codex pending interactions across Pipelines restarts + +The App Server's pending approval/question callback IDs belong to the connection +that received them. Its current official schema exposes no pending-request +listing/replay method. Saving JSON in the Pipelines database cannot make an old +callback answerable on a new upstream connection. + +Added `pipelines-codex-host`: a detached, private local WebSocket bridge that owns +the existing App Server connection. It does not launch Codex. The installed +server scopes hosts by database and upstream socket, finds/reuses the helper, +and initializes only its downstream connection after a restart. The host caches +the one upstream initialization and replays exactly the still-unanswered server +requests. Responses or matching serverRequest/resolved notifications remove +pending entries. Client call IDs are translated to unique host IDs and routed +only to the originating client; replacing a client discards old routes. Client +and callback RPC IDs are capped at 256 bytes. + +Unix sockets are mode 0600 under an owner-verified mode-0700 directory. Per-scope +file locks prevent simultaneous helpers. The existing daemon connection, not a +PID or a model statement, owns pending callbacks. Pending memory is capped at +128 requests / 8 MiB; calls at 256; incoming frames at 16 MiB. Overflow closes +rather than answering an interaction. With no Pipelines client, helpers expire +after 20 minutes with no pending callbacks, or after 24 hours even when pending. +Ordinary notifications are not spooled; current thread readback repairs gaps. + +Native installation, tester packages and Docker include the helper; Go-bin +uninstallation verifies and removes its executable. Missing helper or explicit +PIPELINES_CODEX_HOST=off retains direct mode and logs its restart limitation. +Direct-mode fixtures still pass. Hosts run independently of a launching request's +context and are not killed by installing a replacement binary. + +Verification: +- `go test -race ./internal/codexhost ./internal/codex ./internal/installation + ./cmd/pipelines-server` passes. Real Unix-socket fixtures cover pending approval + and question responses across client replacement, duplicate response rejection, + resolved callbacks, matching thread ownership, old/new RPC-ID collision + isolation, detached arrival/replay, bounded pending state, singleton ownership, + private permissions, shutdown cleanup, and actual helper process spawning/reuse. +- The actual Codex adapter's Open -> Detach -> Open integration retains both + permission and question maps, answers their original IDs, initializes upstream + once, and does not replay resolved requests on a third client. +- Existing direct Codex tests pass; `go build ./...`, `go vet ./...`, and all 53 + native-installer Python tests pass. Full Go integration is the manager's check. + +No live service, conversation, daemon, credential or deployment was changed. +The initial deployment cannot rescue callbacks already owned by the old direct +connection; pending interactions must settle before that first replacement. +A helper crash, upstream App Server restart, or expiry breaks callback identity +and remains an explicit limitation. No disk reconstruction is claimed. diff --git a/internal/codex/chat.go b/internal/codex/chat.go index 13de0c1a..906fbede 100644 --- a/internal/codex/chat.go +++ b/internal/codex/chat.go @@ -4,7 +4,8 @@ // normally reached over a Unix WebSocket socket (the desktop app and the CLI use // the same control socket). This adapter owns one client connection and mirrors // the protocol into the Pipelines chat frames; it never starts a second Codex -// daemon or changes the operator's Codex configuration. +// daemon or changes the operator's Codex configuration. Installed servers use +// a private connection host to retain pending interactions during restarts. package codex import ( @@ -116,6 +117,11 @@ func Open(ctx context.Context) (*Chat, error) { if socket == "" { return nil, errors.New("Codex home is unavailable") } + var socketErr error + socket, socketErr = connectionSocket(ctx, socket) + if socketErr != nil { + return nil, fmt.Errorf("attach Codex connection host: %w", socketErr) + } transport := &http.Transport{ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", socket) @@ -180,8 +186,8 @@ func (c *Chat) Close() { // in the Codex App Server, not in this client, so closing the control // connection leaves them running; no turn is interrupted. The next server // process reconnects and reconciles each thread through thread/read. An -// approval or question the App Server sent to this connection is bound to it, -// so one still pending at shutdown is not answerable after the restart. +// approval or question is retained by the configured connection host and +// replayed on reattach. Direct mode cannot retain connection-bound callbacks. func (c *Chat) Detach() { c.Close() } func (c *Chat) readLoop() { diff --git a/internal/codex/hosted.go b/internal/codex/hosted.go new file mode 100644 index 00000000..6c89729a --- /dev/null +++ b/internal/codex/hosted.go @@ -0,0 +1,74 @@ +package codex + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "git.a-wan.net/alan/pipelines/internal/codexhost" +) + +var hostConfig struct { + sync.Mutex + dir, binary string +} + +// ConfigureHosts enables a private detachable App Server connection bridge for +// new Chats. Empty dir or PIPELINES_CODEX_HOST=off keeps direct mode for tests +// and installations that intentionally disable this facility. +func ConfigureHosts(dir string) { + hostConfig.Lock() + defer hostConfig.Unlock() + hostConfig.dir, hostConfig.binary = "", "" + if strings.TrimSpace(dir) == "" { + return + } + binary := strings.TrimSpace(os.Getenv("PIPELINES_CODEX_HOST")) + if binary == "off" { + return + } + if binary == "" { + if executable, err := os.Executable(); err == nil { + candidate := filepath.Join(filepath.Dir(executable), "pipelines-codex-host") + if info, err := os.Stat(candidate); err == nil && !info.IsDir() { + binary = candidate + } + } + } + if binary == "" { + binary, _ = exec.LookPath("pipelines-codex-host") + } + if binary == "" { + log.Print("codex chat: pipelines-codex-host was not found; pending approvals and questions cannot survive a server restart") + return + } + hostConfig.dir, hostConfig.binary = dir, binary +} + +// DefaultHostDir scopes bridge ownership to the server's database identity. +func DefaultHostDir(scope string) string { + base, err := os.UserCacheDir() + if err != nil { + return "" + } + if absolute, err := filepath.Abs(scope); err == nil { + scope = absolute + } + sum := sha256.Sum256([]byte(scope)) + return filepath.Join(base, "pipelines", "codex-hosts", hex.EncodeToString(sum[:6])) +} +func connectionSocket(ctx context.Context, upstream string) (string, error) { + hostConfig.Lock() + dir, binary := hostConfig.dir, hostConfig.binary + hostConfig.Unlock() + if dir == "" { + return upstream, nil + } + return codexhost.Ensure(ctx, binary, dir, upstream) +} diff --git a/internal/codex/hosted_test.go b/internal/codex/hosted_test.go new file mode 100644 index 00000000..849c0851 --- /dev/null +++ b/internal/codex/hosted_test.go @@ -0,0 +1,186 @@ +package codex + +import ( + "context" + "encoding/json" + "net" + "net/http" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/codexhost" + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +// Exercise the actual adapter's approval/form maps across a complete Chat +// replacement, not merely a proxy socket reconnect. +func TestHostedDetachRetainsAnswerableInteractions(t *testing.T) { + dir, err := os.MkdirTemp("/tmp", "codex-adapter-host-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + t.Setenv("CODEX_HOME", dir) + if err := os.MkdirAll(filepath.Dir(DefaultSocketPath()), 0o700); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + listener, err := net.Listen("unix", DefaultSocketPath()) + if err != nil { + t.Fatal(err) + } + var mu sync.Mutex + var upstream *websocket.Conn + var initialized atomic.Int32 + responses := make(chan rpcMessage, 8) + readyUpstream := make(chan struct{}) + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + mu.Lock() + upstream = conn + mu.Unlock() + close(readyUpstream) + defer conn.CloseNow() + for { + var msg rpcMessage + if wsjson.Read(ctx, conn, &msg) != nil { + return + } + if msg.Method == "" { + responses <- msg + continue + } + if msg.Method == "initialize" { + initialized.Add(1) + } + mu.Lock() + err = wsjson.Write(ctx, conn, map[string]any{"id": msg.ID, "result": map[string]any{}}) + mu.Unlock() + if err != nil { + return + } + } + })} + go server.Serve(listener) + t.Cleanup(func() { + server.Close() + mu.Lock() + if upstream != nil { + upstream.CloseNow() + } + mu.Unlock() + }) + hostDir := filepath.Join(dir, "private") + ready := make(chan string, 1) + done := make(chan error, 1) + go func() { + done <- codexhost.Serve(ctx, codexhost.Options{Dir: hostDir, Upstream: DefaultSocketPath(), Ready: func(socket string) { ready <- socket }}) + }() + select { + case <-ready: + case err := <-done: + t.Fatalf("host start: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("host not ready") + } + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("host did not close") + } + }) + hostConfig.Lock() + oldDir, oldBinary := hostConfig.dir, hostConfig.binary + hostConfig.dir, hostConfig.binary = hostDir, "unused-existing-host" + hostConfig.Unlock() + t.Cleanup(func() { hostConfig.Lock(); hostConfig.dir, hostConfig.binary = oldDir, oldBinary; hostConfig.Unlock() }) + before, err := Open(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(before.Close) + send := func(raw string) { + t.Helper() + mu.Lock() + defer mu.Unlock() + if err := upstream.Write(ctx, websocket.MessageText, []byte(raw)); err != nil { + t.Fatal(err) + } + } + send(`{"id":7,"method":"item/commandExecution/requestApproval","params":{"threadId":"thread-a","itemId":"command","command":"pwd"}}`) + send(`{"id":8,"method":"item/tool/requestUserInput","params":{"threadId":"thread-a","itemId":"question","questions":[{"id":"q","question":"Continue?","isOther":true}]}}`) + wait := func(c *Chat) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + permissions, pErr := c.Permissions(ctx, "thread-a") + forms, fErr := c.Forms(ctx, "thread-a") + if pErr == nil && fErr == nil && len(permissions) == 1 && len(forms) == 1 { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("pending interactions did not arrive") + } + wait(before) + before.Detach() + after, err := Open(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(after.Close) + wait(after) + permissions, _ := after.Permissions(ctx, "thread-a") + if err := after.Reply(ctx, protocol.ChatReplyRequest{SessionID: "thread-a", RequestID: permissions[0].RequestID, Decision: protocol.ChatDecisionOnce}); err != nil { + t.Fatal(err) + } + if err := after.Answer(ctx, protocol.ChatAnswerRequest{SessionID: "thread-a", FormID: "question", Answers: map[string]any{"q": "yes"}}); err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for range 2 { + select { + case msg := <-responses: + seen[string(msg.ID)] = true + case <-time.After(3 * time.Second): + t.Fatal("answer did not reach original upstream connection") + } + } + if !seen["7"] || !seen["8"] { + t.Fatalf("callback identities changed: %+v", seen) + } + if initialized.Load() != 1 { + t.Fatalf("upstream initialized %d times", initialized.Load()) + } + if err := after.Answer(ctx, protocol.ChatAnswerRequest{SessionID: "thread-a", FormID: "question", Answers: map[string]any{"q": "no"}}); err == nil { + t.Fatal("question was answered twice") + } + // Reattach again and use an RPC response as a read-loop barrier: resolved + // callbacks must not return in the new adapter's maps. + after.Detach() + third, err := Open(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(third.Close) + if _, err := third.call(ctx, "fixture/barrier", json.RawMessage(`{}`)); err != nil { + t.Fatal(err) + } + permissions, _ = third.Permissions(ctx, "thread-a") + forms, _ := third.Forms(ctx, "thread-a") + if len(permissions) != 0 || len(forms) != 0 { + t.Fatalf("resolved interactions replayed: %+v %+v", permissions, forms) + } +} diff --git a/internal/codexhost/client.go b/internal/codexhost/client.go new file mode 100644 index 00000000..2a117741 --- /dev/null +++ b/internal/codexhost/client.go @@ -0,0 +1,86 @@ +package codexhost + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" +) + +// Paths identifies a single upstream connection within one server scope. +func Paths(dir, upstream string) (socket, lock string) { + absolute, err := filepath.Abs(upstream) + if err == nil { + upstream = absolute + } + sum := sha256.Sum256([]byte(upstream)) + key := hex.EncodeToString(sum[:8]) + return filepath.Join(dir, key+".sock"), filepath.Join(dir, key+".lock") +} +func reachable(socket string) bool { + conn, err := net.DialTimeout("unix", socket, 100*time.Millisecond) + if err != nil { + return false + } + conn.Close() + return true +} + +// Ensure starts only the private bridge, never Codex itself, when no bridge is +// already serving this upstream/scope. The detached helper survives its parent. +func Ensure(ctx context.Context, binary, dir, upstream string) (string, error) { + if err := EnsureDir(dir); err != nil { + return "", err + } + socket, _ := Paths(dir, upstream) + if reachable(socket) { + return socket, nil + } + logPath := strings.TrimSuffix(socket, ".sock") + ".log" + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return "", err + } + defer logFile.Close() + cmd := exec.Command(binary, "-dir", dir, "-upstream", upstream) + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + cmd.Stderr = logFile + stdout, err := cmd.StdoutPipe() + if err != nil { + return "", err + } + if err := cmd.Start(); err != nil { + return "", err + } + result := make(chan string, 1) + go func() { line, _ := bufio.NewReader(stdout).ReadString('\n'); result <- strings.TrimSpace(line) }() + timer := time.NewTimer(15 * time.Second) + defer timer.Stop() + failed := func(err error) (string, error) { _ = cmd.Process.Kill(); _ = cmd.Wait(); return "", err } + select { + case <-ctx.Done(): + return failed(ctx.Err()) + case <-timer.C: + return failed(errors.New("Codex host start timed out")) + case line := <-result: + if line == "ready "+socket { + go func() { _ = cmd.Wait() }() + return socket, nil + } + // A racing server may have won the singleton lock and started the host. + _ = cmd.Wait() + if reachable(socket) { + return socket, nil + } + return "", fmt.Errorf("Codex host did not start (%s); log: %s", line, logPath) + } +} diff --git a/internal/codexhost/host.go b/internal/codexhost/host.go new file mode 100644 index 00000000..e38e66ec --- /dev/null +++ b/internal/codexhost/host.go @@ -0,0 +1,260 @@ +// Package codexhost keeps the Codex App Server client connection alive across +// Pipelines restarts. It never starts another Codex daemon. Callback identity +// belongs to this upstream connection, so unanswered requests remain in memory +// here until the next Pipelines client attaches and answers them. +package codexhost + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/coder/websocket" +) + +const maxFrame = 16 << 20 +const maxPending = 128 +const maxPendingBytes = 8 << 20 +const maxCalls = 256 + +type message map[string]json.RawMessage + +type downstream struct { + conn *websocket.Conn + ready bool +} +type route struct { + client *downstream + id json.RawMessage +} +type pending struct { + data []byte + thread string +} + +type bridge struct { + ctx context.Context + cancel context.CancelFunc + upstream *websocket.Conn + mu sync.Mutex // serializes routing and all websocket writes + active *downstream + next uint64 + calls map[string]route + pending map[string]pending + pendingBytes int + initialize message + initialized chan struct{} + detached time.Time +} + +// Options describes one private, database-scoped bridge. MaxDetached bounds its +// lifetime without a Pipelines client, even with an unresolved approval. +type Options struct { + Dir, Upstream string + Idle, MaxDetached time.Duration + Ready func(string) +} + +// Serve owns one upstream socket and serves replacement downstream clients until +// cancellation, upstream failure or detached expiry. A singleton lock prevents +// competing hosts from taking the same socket path. +func Serve(ctx context.Context, options Options) error { + if err := EnsureDir(options.Dir); err != nil { + return err + } + socket, lockPath := Paths(options.Dir, options.Upstream) + lock, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + defer lock.Close() + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + return fmt.Errorf("Codex host already owns this scope: %w", err) + } + defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) + if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { + return err + } + listener, err := net.Listen("unix", socket) + if err != nil { + return err + } + defer listener.Close() + defer os.Remove(socket) + if err := os.Chmod(socket, 0o600); err != nil { + return err + } + upstream, err := Dial(ctx, options.Upstream) + if err != nil { + return fmt.Errorf("connect Codex App Server: %w", err) + } + defer upstream.CloseNow() + child, cancel := context.WithCancel(ctx) + defer cancel() + h := &bridge{ctx: child, cancel: cancel, upstream: upstream, calls: map[string]route{}, pending: map[string]pending{}, initialized: make(chan struct{}), detached: time.Now()} + go h.readUpstream() + init := message{"id": json.RawMessage(`"host-initialize"`), "method": json.RawMessage(`"initialize"`), "params": json.RawMessage(`{"clientInfo":{"name":"pipelines","title":"Pipelines","version":"0.1"}}`)} + if err := h.write(upstream, init); err != nil { + return err + } + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + select { + case <-h.initialized: + case <-child.Done(): + return errors.New("Codex App Server disconnected during initialization") + case <-timer.C: + return errors.New("Codex App Server initialization timed out") + } + if failure := h.initialize["error"]; len(failure) > 0 && string(failure) != "null" { + return errors.New("Codex App Server refused initialization") + } + server := &http.Server{Handler: http.HandlerFunc(h.accept), ReadHeaderTimeout: 5 * time.Second} + defer server.Close() + go func() { <-child.Done(); _ = server.Close() }() + if options.Idle <= 0 { + options.Idle = 20 * time.Minute + } + if options.MaxDetached <= 0 { + options.MaxDetached = 24 * time.Hour + } + go h.expire(options.Idle, options.MaxDetached) + if options.Ready != nil { + options.Ready(socket) + } + err = server.Serve(listener) + h.mu.Lock() + if h.active != nil { + h.active.conn.CloseNow() + } + h.mu.Unlock() + if err == http.ErrServerClosed { + return nil + } + return err +} + +func (h *bridge) write(conn *websocket.Conn, msg message) error { + data, err := json.Marshal(msg) + if err != nil { + return err + } + return h.writeRaw(conn, data) +} +func (h *bridge) writeRaw(conn *websocket.Conn, data []byte) error { + ctx, cancel := context.WithTimeout(h.ctx, 5*time.Second) + defer cancel() + return conn.Write(ctx, websocket.MessageText, data) +} +func (h *bridge) send(client *downstream, msg message) { + if err := h.write(client.conn, msg); err != nil { + client.conn.CloseNow() + } +} +func (h *bridge) accept(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + conn.SetReadLimit(maxFrame) + client := &downstream{conn: conn} + defer conn.CloseNow() + h.mu.Lock() + if h.active != nil { + h.active.conn.CloseNow() + } + // A previous client's in-flight RPC responses are never delivered to its + // replacement, even when both start their request counters at one. + h.calls = map[string]route{} + h.active = client + h.mu.Unlock() + defer func() { + h.mu.Lock() + if h.active == client { + h.active = nil + h.detached = time.Now() + h.calls = map[string]route{} + } + h.mu.Unlock() + }() + for { + _, data, err := conn.Read(h.ctx) + if err != nil { + return + } + var msg message + if json.Unmarshal(data, &msg) != nil { + return + } + h.mu.Lock() + if h.active != client { + h.mu.Unlock() + return + } + err = h.fromClient(client, msg) + h.mu.Unlock() + if err != nil { + return + } + } +} + +func (h *bridge) expire(idle, maxDetached time.Duration) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-h.ctx.Done(): + return + case <-ticker.C: + h.mu.Lock() + expired := h.active == nil && (time.Since(h.detached) >= maxDetached || (len(h.pending) == 0 && time.Since(h.detached) >= idle)) + h.mu.Unlock() + if expired { + h.cancel() + return + } + } + } +} + +// Dial opens a WebSocket over an existing local Unix control socket. +func Dial(ctx context.Context, socket string) (*websocket.Conn, error) { + transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socket) + }} + conn, _, err := websocket.Dial(ctx, "ws://localhost/", &websocket.DialOptions{HTTPClient: &http.Client{Transport: transport}}) + if err != nil { + return nil, err + } + conn.SetReadLimit(maxFrame) + return conn, nil +} + +func EnsureDir(dir string) error { + if !filepath.IsAbs(dir) { + return errors.New("Codex host directory must be absolute") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + info, err := os.Lstat(dir) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("Codex host directory is not a real directory") + } + if owner, ok := info.Sys().(*syscall.Stat_t); !ok || owner.Uid != uint32(os.Geteuid()) { + return errors.New("Codex host directory has another owner") + } + return os.Chmod(dir, 0o700) +} diff --git a/internal/codexhost/host_test.go b/internal/codexhost/host_test.go new file mode 100644 index 00000000..661b4328 --- /dev/null +++ b/internal/codexhost/host_test.go @@ -0,0 +1,241 @@ +package codexhost + +import ( + "context" + "encoding/json" + "net" + "net/http" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/coder/websocket" +) + +type fakeServer struct { + ctx context.Context + conn *websocket.Conn + mu sync.Mutex + received chan message + ready chan struct{} +} + +func testHost(t *testing.T) (*fakeServer, string, context.CancelFunc, <-chan error) { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "codex-host-test-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + fake := &fakeServer{ctx: ctx, received: make(chan message, 64), ready: make(chan struct{})} + listener, err := net.Listen("unix", filepath.Join(dir, "upstream.sock")) + if err != nil { + t.Fatal(err) + } + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + fake.mu.Lock() + fake.conn = conn + fake.mu.Unlock() + close(fake.ready) + defer conn.CloseNow() + for { + _, data, err := conn.Read(ctx) + if err != nil { + return + } + var msg message + if json.Unmarshal(data, &msg) != nil { + return + } + var method string + _ = json.Unmarshal(msg["method"], &method) + if method == "initialize" { + fake.send(t, message{"id": msg["id"], "result": json.RawMessage(`{"userAgent":"fixture"}`)}) + } else { + select { + case fake.received <- msg: + case <-ctx.Done(): + return + } + } + } + })} + go server.Serve(listener) + t.Cleanup(func() { + server.Close() + fake.mu.Lock() + if fake.conn != nil { + fake.conn.CloseNow() + } + fake.mu.Unlock() + }) + ready := make(chan string, 1) + done := make(chan error, 1) + go func() { + done <- Serve(ctx, Options{Dir: filepath.Join(dir, "private"), Upstream: filepath.Join(dir, "upstream.sock"), Ready: func(socket string) { ready <- socket }}) + }() + var socket string + select { + case socket = <-ready: + case err := <-done: + t.Fatalf("host failed: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("host did not initialize") + } + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("host did not stop") + } + }) + return fake, socket, cancel, done +} +func (f *fakeServer) send(t *testing.T, msg message) { + t.Helper() + f.mu.Lock() + defer f.mu.Unlock() + data, _ := json.Marshal(msg) + ctx, cancel := context.WithTimeout(f.ctx, 3*time.Second) + defer cancel() + if err := f.conn.Write(ctx, websocket.MessageText, data); err != nil { + t.Fatal(err) + } +} +func (f *fakeServer) next(t *testing.T) message { + t.Helper() + select { + case msg := <-f.received: + return msg + case <-time.After(3 * time.Second): + t.Fatal("missing upstream request") + return nil + } +} +func writeClient(t *testing.T, conn *websocket.Conn, raw string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := conn.Write(ctx, websocket.MessageText, []byte(raw)); err != nil { + t.Fatal(err) + } +} +func readClient(t *testing.T, conn *websocket.Conn) message { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, data, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + var msg message + if err := json.Unmarshal(data, &msg); err != nil { + t.Fatal(err) + } + return msg +} +func attach(t *testing.T, socket string) *websocket.Conn { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := Dial(ctx, socket) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.CloseNow() }) + writeClient(t, conn, `{"id":1,"method":"initialize","params":{"clientInfo":{"name":"pipelines","version":"0.1"}}}`) + if got := readClient(t, conn); string(got["id"]) != "1" || len(got["result"]) == 0 { + t.Fatalf("initialize: %s", got) + } + return conn +} +func method(msg message) string { var s string; _ = json.Unmarshal(msg["method"], &s); return s } + +func TestPendingInteractionsRemainAnswerableAcrossClientReplacement(t *testing.T) { + fake, socket, _, _ := testHost(t) + first := attach(t, socket) + fake.send(t, message{"id": json.RawMessage(`7`), "method": json.RawMessage(`"item/commandExecution/requestApproval"`), "params": json.RawMessage(`{"threadId":"thread-a","itemId":"command","command":"pwd"}`)}) + fake.send(t, message{"id": json.RawMessage(`"question"`), "method": json.RawMessage(`"item/tool/requestUserInput"`), "params": json.RawMessage(`{"threadId":"thread-a","itemId":"question","questions":[{"id":"q","question":"Continue?"}]}`)}) + approval, question := readClient(t, first), readClient(t, first) + first.CloseNow() + second := attach(t, socket) + replay1, replay2 := readClient(t, second), readClient(t, second) + replay := map[string]message{string(replay1["id"]): replay1, string(replay2["id"]): replay2} + for _, original := range []message{approval, question} { + got := replay[string(original["id"])] + if string(got["method"]) != string(original["method"]) || string(got["params"]) != string(original["params"]) { + t.Fatalf("callback changed: %s vs %s", got, original) + } + } + writeClient(t, second, `{"id":7,"result":{"decision":"accept"}}`) + if got := fake.next(t); string(got["id"]) != "7" || string(got["result"]) != `{"decision":"accept"}` { + t.Fatalf("approval answer: %s", got) + } + writeClient(t, second, `{"id":7,"result":{"decision":"decline"}}`) + writeClient(t, second, `{"id":"question","result":{"answers":{"q":{"answers":["yes"]}}}}`) + if got := fake.next(t); string(got["id"]) != `"question"` { + t.Fatalf("duplicate approval reached upstream: %s", got) + } + writeClient(t, second, `{"id":2,"method":"barrier"}`) + if got := fake.next(t); method(got) != "barrier" { + t.Fatalf("duplicate question reached upstream: %s", got) + } + second.CloseNow() + third := attach(t, socket) + writeClient(t, third, `{"id":2,"method":"last"}`) + call := fake.next(t) + fake.send(t, message{"id": call["id"], "result": json.RawMessage(`"done"`)}) + if got := readClient(t, third); string(got["id"]) != "2" || string(got["result"]) != `"done"` { + t.Fatalf("resolved callback was replayed: %s", got) + } +} + +func TestOldRPCResponseCannotAnswerNewClientCall(t *testing.T) { + fake, socket, _, _ := testHost(t) + first := attach(t, socket) + writeClient(t, first, `{"id":2,"method":"old"}`) + old := fake.next(t) + second := attach(t, socket) // closes the obsolete client + writeClient(t, second, `{"id":2,"method":"new"}`) + fresh := fake.next(t) + if string(old["id"]) == string(fresh["id"]) { + t.Fatal("client RPC IDs were reused upstream") + } + fake.send(t, message{"id": old["id"], "result": json.RawMessage(`"old"`)}) + fake.send(t, message{"id": fresh["id"], "result": json.RawMessage(`"new"`)}) + if got := readClient(t, second); string(got["id"]) != "2" || string(got["result"]) != `"new"` { + t.Fatalf("old RPC response crossed client identity: %s", got) + } +} + +func TestResolvedNotificationClearsOnlyMatchingThread(t *testing.T) { + fake, socket, _, _ := testHost(t) + first := attach(t, socket) + request := message{"id": json.RawMessage(`3`), "method": json.RawMessage(`"item/commandExecution/requestApproval"`), "params": json.RawMessage(`{"threadId":"owner","itemId":"command"}`)} + fake.send(t, request) + readClient(t, first) + fake.send(t, message{"method": json.RawMessage(`"serverRequest/resolved"`), "params": json.RawMessage(`{"threadId":"foreign","requestId":3}`)}) + readClient(t, first) + second := attach(t, socket) + if got := readClient(t, second); string(got["id"]) != "3" { + t.Fatalf("foreign thread resolved callback: %s", got) + } + fake.send(t, message{"method": json.RawMessage(`"serverRequest/resolved"`), "params": json.RawMessage(`{"threadId":"owner","requestId":3}`)}) + readClient(t, second) + third := attach(t, socket) + writeClient(t, third, `{"id":2,"method":"barrier"}`) + call := fake.next(t) + fake.send(t, message{"id": call["id"], "result": json.RawMessage(`null`)}) + if got := readClient(t, third); len(got["method"]) > 0 { + t.Fatalf("resolved callback replayed: %s", got) + } +} diff --git a/internal/codexhost/lifecycle_test.go b/internal/codexhost/lifecycle_test.go new file mode 100644 index 00000000..d5ac1e08 --- /dev/null +++ b/internal/codexhost/lifecycle_test.go @@ -0,0 +1,75 @@ +package codexhost + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" +) + +func waitRemoved(t *testing.T, path string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("host socket was not removed: %s", path) +} +func TestPrivateSingletonScopeAndCleanup(t *testing.T) { + _, socket, cancel, _ := testHost(t) + dir := filepath.Dir(socket) + for path, want := range map[string]os.FileMode{dir: 0o700, socket: 0o600} { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != want { + t.Fatalf("%s permissions %o, want %o", path, info.Mode().Perm(), want) + } + } + upstream := filepath.Join(filepath.Dir(dir), "upstream.sock") + if err := Serve(context.Background(), Options{Dir: dir, Upstream: upstream}); err == nil { + t.Fatal("second host stole active singleton") + } + other, _ := Paths(dir, upstream+".other") + if other == socket { + t.Fatal("upstream identities share a socket") + } + cancel() + waitRemoved(t, socket) +} +func TestPendingCallbacksAreRetainedWhileDetachedAndBounded(t *testing.T) { + fake, socket, _, _ := testHost(t) + // No downstream is attached while these arrive: they must still be answerable. + for i := 0; i < maxPending; i++ { + fake.send(t, message{"id": json.RawMessage(fmt.Sprint(i)), "method": json.RawMessage(`"item/tool/requestUserInput"`), "params": json.RawMessage(`{"threadId":"thread"}`)}) + } + client := attach(t, socket) + for range maxPending { + readClient(t, client) + } + client.CloseNow() + fake.send(t, message{"id": json.RawMessage(`999`), "method": json.RawMessage(`"item/tool/requestUserInput"`), "params": json.RawMessage(`{"threadId":"thread"}`)}) + waitRemoved(t, socket) + select { + case got := <-fake.received: + t.Fatalf("host auto-answered on overflow: %s", got) + default: + } +} +func TestEnsureDirRejectsSymlink(t *testing.T) { + dir := t.TempDir() + link := filepath.Join(dir, "link") + if err := os.Symlink(dir, link); err != nil { + t.Fatal(err) + } + if err := EnsureDir(link); err == nil { + t.Fatal("accepted symlink host directory") + } +} diff --git a/internal/codexhost/main.go b/internal/codexhost/main.go new file mode 100644 index 00000000..8b8e8755 --- /dev/null +++ b/internal/codexhost/main.go @@ -0,0 +1,45 @@ +package codexhost + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "time" +) + +// Main is the detached bridge's command entry point. +func Main(args []string) int { + flags := flag.NewFlagSet("pipelines-codex-host", flag.ContinueOnError) + dir := flags.String("dir", "", "private per-server host directory") + upstream := flags.String("upstream", "", "existing Codex App Server Unix socket") + idle := flags.Duration("idle", 20*time.Minute, "exit after this long detached without pending interactions") + maxDetached := flags.Duration("max-detached", 24*time.Hour, "maximum detached lifetime, including pending interactions") + if err := flags.Parse(args); err != nil { + return 2 + } + if *dir == "" || *upstream == "" { + fmt.Fprintln(os.Stderr, "-dir and -upstream are required") + return 2 + } + syscall.Umask(0o077) + signal.Ignore(syscall.SIGHUP, syscall.SIGPIPE) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + err := Serve(ctx, Options{Dir: *dir, Upstream: *upstream, Idle: *idle, MaxDetached: *maxDetached, Ready: func(socket string) { + fmt.Fprintln(os.Stdout, "ready "+socket) + if null, err := os.OpenFile(os.DevNull, os.O_RDWR, 0); err == nil { + _ = syscall.Dup2(int(null.Fd()), 0) + _ = syscall.Dup2(int(null.Fd()), 1) + null.Close() + } + }}) + if err != nil { + fmt.Fprintln(os.Stderr, err) + fmt.Fprintln(os.Stdout, "error "+err.Error()) + return 1 + } + return 0 +} diff --git a/internal/codexhost/process_test.go b/internal/codexhost/process_test.go new file mode 100644 index 00000000..e9647bb8 --- /dev/null +++ b/internal/codexhost/process_test.go @@ -0,0 +1,109 @@ +package codexhost + +import ( + "context" + "encoding/json" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +func TestHostHelperProcess(t *testing.T) { + if os.Getenv("PIPELINES_TEST_CODEX_HOST") != "1" { + return + } + for index, arg := range os.Args { + if arg == "--" { + os.Exit(Main(os.Args[index+1:])) + } + } + os.Exit(2) +} + +func TestEnsureSpawnsReusableDetachedHost(t *testing.T) { + dir, err := os.MkdirTemp("/tmp", "codex-host-process-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + upstream := filepath.Join(dir, "upstream.sock") + listener, err := net.Listen("unix", upstream) + if err != nil { + t.Fatal(err) + } + var original *websocket.Conn + accepted := make(chan *websocket.Conn, 2) + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + accepted <- conn + defer conn.CloseNow() + for { + var msg message + if wsjson.Read(ctx, conn, &msg) != nil { + return + } + if err := wsjson.Write(ctx, conn, message{"id": msg["id"], "result": json.RawMessage(`{}`)}); err != nil { + return + } + } + })} + go server.Serve(listener) + t.Cleanup(func() { + cancel() + server.Close() + if original != nil { + original.CloseNow() + } + }) + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + wrapper := filepath.Join(dir, "helper") + command := "#!/bin/sh\nexec '" + strings.ReplaceAll(executable, "'", "'\\''") + "' -test.run '^TestHostHelperProcess$' -- \"$@\"\n" + if err := os.WriteFile(wrapper, []byte(command), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PIPELINES_TEST_CODEX_HOST", "1") + hostDir := filepath.Join(dir, "private") + startCtx, startCancel := context.WithCancel(context.Background()) + socket, err := Ensure(startCtx, wrapper, hostDir, upstream) + if err != nil { + t.Fatal(err) + } + select { + case original = <-accepted: + case <-time.After(3 * time.Second): + t.Fatal("helper did not attach upstream") + } + startCancel() // completion of the launching request must not kill the host + first := attach(t, socket) + first.CloseNow() + reused, err := Ensure(context.Background(), "/missing/helper", hostDir, upstream) + if err != nil || reused != socket { + t.Fatalf("live host was not reused: %s %v", reused, err) + } + second := attach(t, socket) + second.CloseNow() + select { + case extra := <-accepted: + extra.CloseNow() + t.Fatal("reconnect started a second upstream connection") + default: + } + // Closing the existing upstream daemon connection must release the bridge. + original.CloseNow() + waitRemoved(t, socket) +} diff --git a/internal/codexhost/rpc.go b/internal/codexhost/rpc.go new file mode 100644 index 00000000..b2dd28ad --- /dev/null +++ b/internal/codexhost/rpc.go @@ -0,0 +1,159 @@ +package codexhost + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "sort" +) + +func (h *bridge) fromClient(client *downstream, msg message) error { + var method string + _ = json.Unmarshal(msg["method"], &method) + id := msg["id"] + if len(id) > 256 { + return errors.New("Codex RPC id exceeds bridge limit") + } + if method == "initialize" && len(id) > 0 { + if client.ready { + return errors.New("duplicate initialize") + } + response := message{"id": id} + for key, value := range h.initialize { + if key != "id" { + response[key] = value + } + } + if err := h.write(client.conn, response); err != nil { + return err + } + client.ready = true + keys := make([]string, 0, len(h.pending)) + for key := range h.pending { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if err := h.writeRaw(client.conn, h.pending[key].data); err != nil { + return err + } + } + return nil + } + if !client.ready { + return errors.New("initialize required") + } + if method == "initialized" { + return nil + } // the host initialized upstream once + if method != "" && len(id) > 0 { + if len(h.calls) >= maxCalls { + return errors.New("too many outstanding Codex calls") + } + h.next++ + hostID := json.RawMessage(fmt.Sprintf(`"host-%d"`, h.next)) + h.calls[string(hostID)] = route{client, id} + msg["id"] = hostID + if err := h.write(h.upstream, msg); err != nil { + h.cancel() + return err + } + return nil + } + if method == "" && len(id) > 0 { + key := string(id) + item, ok := h.pending[key] + if !ok { + return nil + } // stale/duplicate callback response is never replayed + if err := h.write(h.upstream, msg); err != nil { + h.cancel() + return err + } + delete(h.pending, key) + h.pendingBytes -= len(item.data) + return nil + } + if err := h.write(h.upstream, msg); err != nil { + h.cancel() + return err + } + return nil +} + +func (h *bridge) readUpstream() { + defer h.cancel() + for { + _, data, err := h.upstream.Read(h.ctx) + if err != nil { + if h.ctx.Err() == nil { + log.Print("Codex App Server connection ended; connection-bound interactions cannot be recovered by a new upstream connection") + } + return + } + var msg message + if json.Unmarshal(data, &msg) != nil { + return + } + h.mu.Lock() + ok := h.fromUpstream(msg, data) + h.mu.Unlock() + if !ok { + log.Print("Codex bridge received invalid or over-budget state; closing rather than auto-answering interactions") + return + } + } +} +func (h *bridge) fromUpstream(msg message, data []byte) bool { + id := msg["id"] + if len(id) > 256 { + return false + } + var method string + _ = json.Unmarshal(msg["method"], &method) + if string(id) == `"host-initialize"` && method == "" { + if h.initialize != nil { + return false + } + h.initialize = msg + close(h.initialized) + return true + } + if len(id) > 0 && method != "" { + key := string(id) + old, exists := h.pending[key] + if (!exists && len(h.pending) >= maxPending) || h.pendingBytes-len(old.data)+len(data) > maxPendingBytes { + return false + } + var params struct { + Thread string `json:"threadId"` + } + _ = json.Unmarshal(msg["params"], ¶ms) + h.pending[key] = pending{append([]byte(nil), data...), params.Thread} + h.pendingBytes += len(data) - len(old.data) + } else if len(id) > 0 { + target, ok := h.calls[string(id)] + delete(h.calls, string(id)) + if ok && h.active == target.client && target.client.ready { + msg["id"] = target.id + h.send(target.client, msg) + } + return true + } else if method == "serverRequest/resolved" { + var p struct { + Thread string `json:"threadId"` + ID json.RawMessage `json:"requestId"` + } + if json.Unmarshal(msg["params"], &p) == nil { + if old, ok := h.pending[string(p.ID)]; ok && old.thread == p.Thread { + delete(h.pending, string(p.ID)) + h.pendingBytes -= len(old.data) + } + } + } + if h.active != nil && h.active.ready { + h.send(h.active, msg) + } + return true +} diff --git a/internal/installation/uninstall.go b/internal/installation/uninstall.go index c27634bd..acdf6d6b 100644 --- a/internal/installation/uninstall.go +++ b/internal/installation/uninstall.go @@ -40,7 +40,7 @@ func Uninstall(executable string, args []string, in io.Reader, out io.Writer) er } dir := filepath.Dir(resolved) var files []string - for _, command := range []string{"pipelines", "pipelines-server", "pipelines-tui", "pipelines-claude-host"} { + for _, command := range []string{"pipelines", "pipelines-server", "pipelines-tui", "pipelines-claude-host", "pipelines-codex-host"} { path := filepath.Join(dir, command) info, err := buildinfo.ReadFile(path) if os.IsNotExist(err) { diff --git a/scripts/package.py b/scripts/package.py index 39a47046..37511ce9 100755 --- a/scripts/package.py +++ b/scripts/package.py @@ -36,7 +36,7 @@ def package(output, version, target): if system not in ("darwin", "linux") or arch not in ("arm64", "amd64"): raise ValueError("Supported: source, darwin-arm64/amd64, linux-arm64/amd64") env = dict(os.environ, GOOS=system, GOARCH=arch, CGO_ENABLED="0") - for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines"), ("pipelines-claude-host", "pipelines-claude-host")): + for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines"), ("pipelines-claude-host", "pipelines-claude-host"), ("pipelines-codex-host", "pipelines-codex-host")): subprocess.run(["go", "build", "-trimpath", "-o", str(binaries / binary), "./cmd/" + command], cwd=ROOT, env=env, check=True) (binaries / "platform.json").write_text(json.dumps({"os": "Darwin" if system == "darwin" else "Linux", "arch": "arm64" if arch == "arm64" and system == "darwin" else "aarch64" if arch == "arm64" else "x86_64"})) diff --git a/scripts/setup-native.py b/scripts/setup-native.py index ba29b0f7..fd15ffcb 100644 --- a/scripts/setup-native.py +++ b/scripts/setup-native.py @@ -448,9 +448,9 @@ class Setup: # --- build and skill ------------------------------------------------- - # pipelines-claude-host keeps Claude Code chats alive across server - # restarts; the server finds it next to its own executable. - COMMANDS = ("pipelines-server", "pipelines", "pipelines-claude-host") + # Detachable hosts retain Claude processes and Codex interaction callbacks + # across server restarts; the server finds them beside its own executable. + COMMANDS = ("pipelines-server", "pipelines", "pipelines-claude-host", "pipelines-codex-host") def build(self, prefix): """Builds the server and CLI into bin/.next; install_binaries moves diff --git a/scripts/test_setup_native.py b/scripts/test_setup_native.py index d65aa9e2..1e1561f5 100644 --- a/scripts/test_setup_native.py +++ b/scripts/test_setup_native.py @@ -305,7 +305,7 @@ class SetupNativeTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "Node.js 18"): installer.configure_web(Path(directory) / "prefix", setup_native.arguments([])) - def test_build_installs_only_server_cli_and_claude_host_with_trimpath(self): + def test_build_installs_server_cli_and_detachable_hosts_with_trimpath(self): with tempfile.TemporaryDirectory() as directory: recorder = Recorder() installer = self.installer(directory, recorder) @@ -313,12 +313,13 @@ class SetupNativeTests(unittest.TestCase): installer.prepare_layout(prefix) installer.build(prefix) builds = [call for call in recorder.calls if call[:2] == ["go", "build"]] - self.assertEqual(len(builds), 3) + self.assertEqual(len(builds), 4) for call in builds: self.assertIn("-trimpath", call) self.assertTrue(any(call[-1] == "./cmd/pipelines-server" for call in builds)) self.assertTrue(any(call[-1] == "./cmd/pipelines" for call in builds)) self.assertTrue(any(call[-1] == "./cmd/pipelines-claude-host" for call in builds)) + self.assertTrue(any(call[-1] == "./cmd/pipelines-codex-host" for call in builds)) self.assertFalse(any("pipelines-tui" in part for call in recorder.calls for part in call)) def test_layout_creates_private_prefix_and_copies_manage(self):