c810437aa5
A pipelines-claude-host process owns each claude -p CLI and its pipes, spools its stream-json output with offsets and serves a private Unix socket. The server detaches on shutdown and reattaches on start, replaying the spool to rebuild turns, approvals, questions, background tasks and the live model. Explicit close, idle release and CLI exit stop the host; a crashed host falls back to --resume with a notice.
386 lines
13 KiB
Go
386 lines
13 KiB
Go
package claude
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/claudehost"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// The test binary doubles as pipelines-claude-host: a host spawned by a test
|
|
// re-executes it with this variable set.
|
|
const hostSelfTest = "PIPELINES_CLAUDE_HOST_SELFTEST"
|
|
|
|
func TestMain(m *testing.M) {
|
|
if os.Getenv(hostSelfTest) == "1" {
|
|
os.Exit(claudehost.Main(os.Args[1:]))
|
|
}
|
|
os.Exit(m.Run())
|
|
}
|
|
|
|
type hostedFixture struct {
|
|
hostDir string
|
|
gate string
|
|
logPath string
|
|
}
|
|
|
|
// hostedSetup configures the fake CLI and detachable hosts in a short private
|
|
// directory (Unix socket paths are length-limited).
|
|
func hostedSetup(t *testing.T) hostedFixture {
|
|
t.Helper()
|
|
base, err := os.MkdirTemp("/tmp", "pch")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
source, err := os.ReadFile(filepath.Join("testdata", "fake-claude.py"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
script := filepath.Join(base, "claude")
|
|
if err := os.WriteFile(script, source, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executable, err := os.Executable()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture := hostedFixture{hostDir: filepath.Join(base, "h"), gate: filepath.Join(base, "gate"), logPath: filepath.Join(base, "args.log")}
|
|
t.Setenv("PIPELINES_CLAUDE", script)
|
|
t.Setenv("CLAUDE_CONFIG_DIR", filepath.Join(base, "config"))
|
|
t.Setenv("FAKE_CLAUDE_LOG", fixture.logPath)
|
|
t.Setenv("FAKE_CLAUDE_GATE", fixture.gate)
|
|
t.Setenv("PIPELINES_CLAUDE_HOST", executable)
|
|
t.Setenv(hostSelfTest, "1")
|
|
ConfigureHosts(fixture.hostDir)
|
|
t.Cleanup(func() {
|
|
ConfigureHosts("")
|
|
for _, record := range claudehost.Records(fixture.hostDir) {
|
|
claudehost.KillStale(fixture.hostDir, record)
|
|
}
|
|
_ = os.RemoveAll(base)
|
|
})
|
|
return fixture
|
|
}
|
|
|
|
func hostedChat(t *testing.T) *Chat {
|
|
t.Helper()
|
|
chat := New(context.Background())
|
|
if chat.hostDir == "" {
|
|
t.Fatal("hosting is not configured")
|
|
}
|
|
t.Cleanup(chat.Close)
|
|
return chat
|
|
}
|
|
|
|
func onlyRecord(t *testing.T, fixture hostedFixture) claudehost.Record {
|
|
t.Helper()
|
|
var records []claudehost.Record
|
|
eventually(t, "a host registry record", func() bool {
|
|
records = claudehost.Records(fixture.hostDir)
|
|
return len(records) == 1
|
|
})
|
|
return records[0]
|
|
}
|
|
|
|
func alive(pid int) bool { return pid > 0 && syscall.Kill(pid, 0) == nil }
|
|
|
|
func textFrame(contains string) func(protocol.ChatEventFrame) bool {
|
|
return func(frame protocol.ChatEventFrame) bool {
|
|
return strings.Contains(frame.Delta, contains) || (frame.Part != nil && strings.Contains(frame.Part.Text, contains))
|
|
}
|
|
}
|
|
|
|
func stepStarted(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepStarted
|
|
}
|
|
|
|
// restart stands in for a server restart: the old Chat detaches (its host
|
|
// keeps running) and a new one is built over the same host directory.
|
|
func restart(t *testing.T, old *Chat) *Chat {
|
|
t.Helper()
|
|
old.Detach()
|
|
return hostedChat(t)
|
|
}
|
|
|
|
func TestHostedTurnSurvivesRestartAndReplaysMissedOutput(t *testing.T) {
|
|
fixture := hostedSetup(t)
|
|
first := hostedChat(t)
|
|
session := newCreatedChat(t, first)
|
|
frames := subscribe(t, first, session.ID)
|
|
if err := first.Prompt(context.Background(), session.ID, "gate please"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "text before the restart", textFrame("Working"))
|
|
record := onlyRecord(t, fixture)
|
|
first.Detach()
|
|
if !alive(record.HostPID) || !alive(record.ClaudePID) {
|
|
t.Fatal("detaching must leave the host and its CLI running")
|
|
}
|
|
|
|
// The CLI finishes its turn while no server is attached.
|
|
if err := os.WriteFile(fixture.gate, nil, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
eventually(t, "the turn result in the spool", func() bool {
|
|
data, _ := os.ReadFile(record.Spool)
|
|
return strings.Contains(string(data), `"type":"result"`)
|
|
})
|
|
|
|
second := newChat(context.Background())
|
|
t.Cleanup(second.Close)
|
|
second.sessionFor(session.ID, session.Directory)
|
|
frames = subscribe(t, second, session.ID)
|
|
second.reattachHosts()
|
|
waitFrame(t, frames, "the restored running turn", stepStarted)
|
|
waitFrame(t, frames, "text produced while disconnected", textFrame("after the restart"))
|
|
waitStepEnded(t, frames)
|
|
if second.TurnRunning(session.ID) {
|
|
t.Fatal("the replayed result must settle the turn")
|
|
}
|
|
if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 {
|
|
t.Fatalf("the CLI must not be restarted, spawns: %+v", spawns)
|
|
}
|
|
// The same CLI keeps serving turns.
|
|
if err := second.Prompt(context.Background(), session.ID, "hello again"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "the next turn's text", textFrame("Hello"))
|
|
waitStepEnded(t, frames)
|
|
}
|
|
|
|
func TestPendingApprovalIsAnsweredAfterRestart(t *testing.T) {
|
|
hostedSetup(t)
|
|
first := hostedChat(t)
|
|
session := newCreatedChat(t, first)
|
|
if err := first.Prompt(context.Background(), session.ID, "permission please"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var before []protocol.ChatPermissionFrame
|
|
eventually(t, "the approval before the restart", func() bool {
|
|
before, _ = first.Permissions(context.Background(), session.ID)
|
|
return len(before) == 1
|
|
})
|
|
second := restart(t, first)
|
|
running, err := second.ProbeTurn(context.Background(), session.ID)
|
|
if err != nil || !running {
|
|
t.Fatalf("the restored turn must be running: %v %v", running, err)
|
|
}
|
|
after, err := second.Permissions(context.Background(), session.ID)
|
|
if err != nil || len(after) != 1 || after[0].RequestID != before[0].RequestID {
|
|
t.Fatalf("the pending approval must survive the restart: %+v %v", after, err)
|
|
}
|
|
frames := subscribe(t, second, session.ID)
|
|
if err := second.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: after[0].RequestID, Decision: protocol.ChatDecisionOnce}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "the approved tool result", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
|
|
frame.Part.State.Status == protocol.ChatToolCompleted && len(frame.Part.State.Content) == 1 &&
|
|
strings.Contains(frame.Part.State.Content[0].Text, "File created")
|
|
})
|
|
waitStepEnded(t, frames)
|
|
|
|
// An answered approval is not shown again by a later restart's replay.
|
|
third := restart(t, second)
|
|
if pending, _ := third.Permissions(context.Background(), session.ID); len(pending) != 0 {
|
|
t.Fatalf("an answered approval reappeared: %+v", pending)
|
|
}
|
|
if third.TurnRunning(session.ID) {
|
|
t.Fatal("a finished turn must not be restored as running")
|
|
}
|
|
}
|
|
|
|
func TestQuestionSurvivesRestart(t *testing.T) {
|
|
hostedSetup(t)
|
|
first := hostedChat(t)
|
|
session := newCreatedChat(t, first)
|
|
if err := first.Prompt(context.Background(), session.ID, "question please"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
eventually(t, "the question before the restart", func() bool {
|
|
forms, _ := first.Forms(context.Background(), session.ID)
|
|
return len(forms) == 1
|
|
})
|
|
second := restart(t, first)
|
|
forms, err := second.Forms(context.Background(), session.ID)
|
|
if err != nil || len(forms) != 1 {
|
|
t.Fatalf("the question must survive the restart: %+v %v", forms, err)
|
|
}
|
|
frames := subscribe(t, second, session.ID)
|
|
if err := second.Answer(context.Background(), protocol.ChatAnswerRequest{SessionID: session.ID, FormID: forms[0].ID, Answers: map[string]any{"q0": "Beta"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "the answered question", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
|
|
len(frame.Part.State.Content) == 1 && strings.Contains(frame.Part.State.Content[0].Text, "Beta")
|
|
})
|
|
waitStepEnded(t, frames)
|
|
}
|
|
|
|
func TestInterruptAfterRestart(t *testing.T) {
|
|
hostedSetup(t)
|
|
first := hostedChat(t)
|
|
session := newCreatedChat(t, first)
|
|
frames := subscribe(t, first, session.ID)
|
|
if err := first.Prompt(context.Background(), session.ID, "long task"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "the long turn's text", textFrame("Hello"))
|
|
second := restart(t, first)
|
|
if running, _ := second.ProbeTurn(context.Background(), session.ID); !running {
|
|
t.Fatal("the long turn must still be running after the restart")
|
|
}
|
|
frames = subscribe(t, second, session.ID)
|
|
if err := second.Interrupt(context.Background(), session.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
if second.TurnRunning(session.ID) {
|
|
t.Fatal("the interrupted turn must settle")
|
|
}
|
|
}
|
|
|
|
func TestModelAndBackgroundTaskSurviveRestart(t *testing.T) {
|
|
fixture := hostedSetup(t)
|
|
first := hostedChat(t)
|
|
session := newCreatedChat(t, first)
|
|
if _, err := first.Models(context.Background(), session.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := first.SwitchModel(context.Background(), session.ID, Backend, "haiku"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
frames := subscribe(t, first, session.ID)
|
|
if err := first.Prompt(context.Background(), session.ID, "background please"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
if shells, _ := first.Shells(context.Background(), session.ID); len(shells) != 1 {
|
|
t.Fatalf("the background command must be running: %+v", shells)
|
|
}
|
|
second := restart(t, first)
|
|
shells, err := second.Shells(context.Background(), session.ID)
|
|
if err != nil || len(shells) != 1 || shells[0].Command != "sleep 600" {
|
|
t.Fatalf("the background command must still be listed after the restart: %+v %v", shells, err)
|
|
}
|
|
s := second.session(session.ID)
|
|
s.mu.Lock()
|
|
applied, models := s.appliedModel, len(s.models)
|
|
s.mu.Unlock()
|
|
if applied != "haiku" || models != 3 {
|
|
t.Fatalf("the live model and catalog must be restored: %q %d", applied, models)
|
|
}
|
|
frames = subscribe(t, second, session.ID)
|
|
if err := second.PromptWithMessageIDModel(context.Background(), session.ID, "notify me", "", "haiku"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
if shells, _ := second.Shells(context.Background(), session.ID); len(shells) != 0 {
|
|
t.Fatalf("the notification must finish the restored command: %+v", shells)
|
|
}
|
|
if spawns := readArgs(t, fixture.logPath); len(spawns) != 1 {
|
|
t.Fatalf("the CLI must not be restarted, spawns: %d", len(spawns))
|
|
}
|
|
}
|
|
|
|
func TestHostCrashFallsBackToResume(t *testing.T) {
|
|
fixture := hostedSetup(t)
|
|
chat := hostedChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
frames := subscribe(t, chat, session.ID)
|
|
if err := chat.Prompt(context.Background(), session.ID, "long task"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "the long turn's text", textFrame("Hello"))
|
|
record := onlyRecord(t, fixture)
|
|
if err := syscall.Kill(record.HostPID, syscall.SIGKILL); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "the host crash notice", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Part != nil && frame.Part.State != nil && frame.Part.State.Error != nil &&
|
|
strings.Contains(frame.Part.State.Error.Message, "--resume")
|
|
})
|
|
waitStepEnded(t, frames)
|
|
eventually(t, "the orphaned CLI to stop", func() bool { return !alive(record.ClaudePID) || isZombie(record.ClaudePID) })
|
|
if _, err := os.Stat(record.Socket); !os.IsNotExist(err) {
|
|
t.Fatalf("the crashed host's files must be removed: %v", err)
|
|
}
|
|
if err := chat.Prompt(context.Background(), session.ID, "hello again"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
spawns := readArgs(t, fixture.logPath)
|
|
if len(spawns) != 2 || argValue(spawns[1].Args, "--resume") != session.ID {
|
|
t.Fatalf("the next prompt must resume the session: %+v", spawns)
|
|
}
|
|
}
|
|
|
|
func isZombie(pid int) bool {
|
|
var status syscall.WaitStatus
|
|
got, _ := syscall.Wait4(pid, &status, syscall.WNOHANG, nil)
|
|
return got == pid
|
|
}
|
|
|
|
func TestReleasingAHostedChatStopsItsHost(t *testing.T) {
|
|
fixture := hostedSetup(t)
|
|
chat := hostedChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
frames := subscribe(t, chat, session.ID)
|
|
if err := chat.Prompt(context.Background(), session.ID, "hello"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
record := onlyRecord(t, fixture)
|
|
if err := chat.ReleaseSession(context.Background(), session.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
eventually(t, "the host to exit", func() bool { return !alive(record.HostPID) })
|
|
eventually(t, "the CLI to exit", func() bool { return !alive(record.ClaudePID) || isZombie(record.ClaudePID) })
|
|
if records := claudehost.Records(fixture.hostDir); len(records) != 0 {
|
|
t.Fatalf("a released host must remove its registry record: %+v", records)
|
|
}
|
|
// A restarted server finds nothing to reattach; the next prompt resumes.
|
|
next := hostedChat(t)
|
|
if next.session(session.ID) != nil {
|
|
t.Fatal("a released session must not be reattached")
|
|
}
|
|
}
|
|
|
|
func TestHostStatusReportsRegistryAndPrivateSocket(t *testing.T) {
|
|
fixture := hostedSetup(t)
|
|
chat := hostedChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
if _, err := chat.Models(context.Background(), session.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record := onlyRecord(t, fixture)
|
|
info, err := os.Stat(record.Socket)
|
|
if err != nil || info.Mode().Perm()&0o077 != 0 {
|
|
t.Fatalf("the socket must be private: %v %v", info.Mode(), err)
|
|
}
|
|
dir, err := os.Stat(fixture.hostDir)
|
|
if err != nil || dir.Mode().Perm() != 0o700 {
|
|
t.Fatalf("the host directory must be private: %v %v", dir.Mode(), err)
|
|
}
|
|
conn, err := claudehost.Dial(record.Socket, time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.Close()
|
|
if err := conn.Send(claudehost.Frame{Op: claudehost.OpStatus}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
frame, err := conn.Recv()
|
|
if err != nil || frame.Status == nil || frame.Status.SessionID != session.ID || len(frame.Status.Init) == 0 {
|
|
raw, _ := json.Marshal(frame)
|
|
t.Fatalf("status: %s %v", raw, err)
|
|
}
|
|
}
|