Files
pipelines/internal/claude/claude_test.go
T

459 lines
15 KiB
Go

package claude
import (
"context"
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func testChat(t *testing.T) *Chat {
t.Helper()
dir := t.TempDir()
source, err := os.ReadFile(filepath.Join("testdata", "fake-claude.py"))
if err != nil {
t.Fatal(err)
}
script := filepath.Join(dir, "claude")
if err := os.WriteFile(script, source, 0o700); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_CLAUDE", script)
t.Setenv("CLAUDE_CONFIG_DIR", filepath.Join(dir, "config"))
t.Setenv("FAKE_CLAUDE_LOG", filepath.Join(dir, "args.log"))
chat := New(context.Background())
t.Cleanup(chat.Close)
return chat
}
func newCreatedChat(t *testing.T, chat *Chat) protocol.ChatSession {
t.Helper()
session, err := chat.Create(context.Background(), "Fixture", t.TempDir())
if err != nil {
t.Fatal(err)
}
return session
}
func subscribe(t *testing.T, chat *Chat, id string) chan protocol.ChatEventFrame {
t.Helper()
frames := make(chan protocol.ChatEventFrame, 512)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
_ = chat.Stream(ctx, id, nil, func(frame protocol.ChatEventFrame) error {
select {
case frames <- frame:
default:
}
return nil
})
}()
t.Cleanup(func() {
cancel()
<-done
})
// Wait for the subscription to be attached before prompting.
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
chat.session(id).mu.Lock()
count := len(chat.session(id).subs)
chat.session(id).mu.Unlock()
if count > 0 {
return frames
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("stream did not attach")
return nil
}
func waitFrame(t *testing.T, frames <-chan protocol.ChatEventFrame, what string, match func(protocol.ChatEventFrame) bool) protocol.ChatEventFrame {
t.Helper()
deadline := time.After(6 * time.Second)
for {
select {
case frame := <-frames:
if match(frame) {
return frame
}
case <-deadline:
t.Fatalf("timed out waiting for %s", what)
}
}
}
func waitStepEnded(t *testing.T, frames <-chan protocol.ChatEventFrame) {
t.Helper()
waitFrame(t, frames, "step ended", func(frame protocol.ChatEventFrame) bool {
return frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded
})
}
func eventually(t *testing.T, what string, check func() bool) {
t.Helper()
deadline := time.Now().Add(6 * time.Second)
for time.Now().Before(deadline) {
if check() {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("timed out waiting for %s", what)
}
func TestTurnStreamsTextWithoutDuplicatingAndReportsUsage(t *testing.T) {
chat := testChat(t)
session := newCreatedChat(t, chat)
frames := subscribe(t, chat, session.ID)
if err := chat.Prompt(context.Background(), session.ID, "hello"); err != nil {
t.Fatal(err)
}
text := ""
var usage *protocol.ChatUsage
ended := false
deadline := time.After(8 * time.Second)
for !ended || usage == nil {
select {
case frame := <-frames:
switch frame.Kind {
case protocol.ChatPartText:
text += frame.Delta
case protocol.ChatEventStep:
if frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded {
ended = true
}
case protocol.ChatEventUsage:
usage = frame.Usage
}
case <-deadline:
t.Fatalf("turn did not settle (ended=%v usage=%v text=%q)", ended, usage != nil, text)
}
}
if text != "Hello world" {
t.Fatalf("streamed text = %q, want one copy of Hello world", text)
}
if usage.TotalTokens == nil || *usage.TotalTokens != 100 {
t.Fatalf("usage totals: %+v", usage)
}
if usage.ContextWindow == nil || *usage.ContextWindow != 200000 || usage.ContextTokens == nil || *usage.ContextTokens != 60 {
t.Fatalf("context usage: %+v", usage)
}
if usage.CostUSD == nil || *usage.CostUSD != 0.25 || usage.Scope != "session" {
t.Fatalf("cost/scope: %+v", usage)
}
}
func TestApprovalAllowAndRejectReachTheCLI(t *testing.T) {
chat := testChat(t)
session := newCreatedChat(t, chat)
frames := subscribe(t, chat, session.ID)
if err := chat.Prompt(context.Background(), session.ID, "permission please"); err != nil {
t.Fatal(err)
}
waitFrame(t, frames, "running tool", func(frame protocol.ChatEventFrame) bool {
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.Name == "Write" && frame.Part.State != nil && frame.Part.State.Status == protocol.ChatToolRunning
})
var pending []protocol.ChatPermissionFrame
eventually(t, "permission request", func() bool {
var err error
pending, err = chat.Permissions(context.Background(), session.ID)
return err == nil && len(pending) == 1
})
request := pending[0]
if request.Action != "Write" || len(request.Resources) != 1 || !strings.HasSuffix(request.Resources[0], "note.txt") {
t.Fatalf("approval frame: %+v", request)
}
if request.AlwaysLabel == "" {
t.Fatal("a suggestion should offer Always allow")
}
if err := chat.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: request.RequestID, Decision: protocol.ChatDecisionOnce}); err != nil {
t.Fatal(err)
}
waitFrame(t, frames, "completed tool", func(frame protocol.ChatEventFrame) bool {
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.ID == "toolu_fake_1" &&
frame.Part.State != nil && frame.Part.State.Status == protocol.ChatToolCompleted &&
len(frame.Part.State.Content) == 1 && strings.Contains(frame.Part.State.Content[0].Text, "File created")
})
waitStepEnded(t, frames)
// A rejection is delivered as a deny, and the turn still settles.
if err := chat.Prompt(context.Background(), session.ID, "permission reject"); err != nil {
t.Fatal(err)
}
eventually(t, "second permission request", func() bool {
var err error
pending, err = chat.Permissions(context.Background(), session.ID)
return err == nil && len(pending) == 1
})
if err := chat.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: pending[0].RequestID, Decision: protocol.ChatDecisionReject}); err != nil {
t.Fatal(err)
}
waitFrame(t, frames, "rejected tool", func(frame protocol.ChatEventFrame) bool {
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
frame.Part.State.Status == protocol.ChatToolError && frame.Part.State.Error != nil
})
waitStepEnded(t, frames)
if remaining, _ := chat.Permissions(context.Background(), session.ID); len(remaining) != 0 {
t.Fatalf("resolved approval still pending: %+v", remaining)
}
}
func TestQuestionBecomesAFormAndAnswerReturnsToCLI(t *testing.T) {
chat := testChat(t)
session := newCreatedChat(t, chat)
frames := subscribe(t, chat, session.ID)
if err := chat.Prompt(context.Background(), session.ID, "question please"); err != nil {
t.Fatal(err)
}
var forms []protocol.ChatForm
eventually(t, "question form", func() bool {
var err error
forms, err = chat.Forms(context.Background(), session.ID)
return err == nil && len(forms) == 1
})
form := forms[0]
if form.Kind != "question" || len(form.Fields) != 1 || len(form.Fields[0].Options) != 2 {
t.Fatalf("form: %+v", form)
}
if err := chat.Answer(context.Background(), protocol.ChatAnswerRequest{SessionID: session.ID, FormID: form.ID, Answers: map[string]any{"q0": "Alpha"}}); err != nil {
t.Fatal(err)
}
waitFrame(t, frames, "answered tool", func(frame protocol.ChatEventFrame) bool {
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
frame.Part.State.Status == protocol.ChatToolCompleted && len(frame.Part.State.Content) == 1 &&
strings.Contains(frame.Part.State.Content[0].Text, "Alpha")
})
waitStepEnded(t, frames)
if forms, _ := chat.Forms(context.Background(), session.ID); len(forms) != 0 {
t.Fatalf("resolved form still pending: %+v", forms)
}
}
func TestInterruptSettlesTheTurn(t *testing.T) {
chat := testChat(t)
session := newCreatedChat(t, chat)
frames := subscribe(t, chat, session.ID)
if err := chat.Prompt(context.Background(), session.ID, "long task"); err != nil {
t.Fatal(err)
}
if err := chat.Interrupt(context.Background(), session.ID); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
chat.session(session.ID).mu.Lock()
running := chat.session(session.ID).running
chat.session(session.ID).mu.Unlock()
if running {
t.Fatal("session still marked running after interrupt")
}
}
func TestModelChoiceIsAppliedAndProcessResumes(t *testing.T) {
chat := testChat(t)
session := newCreatedChat(t, chat)
if err := chat.SwitchModel(context.Background(), session.ID, Backend, "sonnet"); err != nil {
t.Fatal(err)
}
models, err := chat.Models(context.Background(), session.ID)
if err != nil {
t.Fatal(err)
}
if len(models) != 3 || models[1].ID != "sonnet" || models[1].Provider != Backend {
t.Fatalf("models: %+v", models)
}
frames := subscribe(t, chat, session.ID)
if err := chat.Prompt(context.Background(), session.ID, "hello"); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
logPath := os.Getenv("FAKE_CLAUDE_LOG")
entries := readArgs(t, logPath)
if argValue(entries[len(entries)-1].Args, "--model") != "sonnet" {
t.Fatalf("spawn did not apply the selected model: %+v", entries)
}
if argValue(entries[len(entries)-1].Args, "--session-id") != session.ID {
t.Fatalf("new session did not pin its id: %+v", entries)
}
// Releasing the process and prompting again must resume the same session.
chat.session(session.ID).closeProcess()
if err := chat.Prompt(context.Background(), session.ID, "hello again"); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
entries = readArgs(t, logPath)
if len(entries) < 2 || argValue(entries[len(entries)-1].Args, "--resume") != session.ID {
t.Fatalf("restart did not resume the session: %+v", entries)
}
}
type argsEntry struct {
Args []string `json:"args"`
Cwd string `json:"cwd"`
}
func readArgs(t *testing.T, path string) []argsEntry {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var entries []argsEntry
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
if line == "" {
continue
}
var entry argsEntry
if json.Unmarshal([]byte(line), &entry) == nil {
entries = append(entries, entry)
}
}
return entries
}
func argValue(values []string, flag string) string {
for index, value := range values {
if value == flag && index+1 < len(values) {
return values[index+1]
}
}
return ""
}
func TestHistoryMergesBlocksAndMapsPromptIdentity(t *testing.T) {
chat := testChat(t)
session := newCreatedChat(t, chat)
frames := subscribe(t, chat, session.ID)
if err := chat.PromptWithMessageID(context.Background(), session.ID, "permission please", "native-user-1"); err != nil {
t.Fatal(err)
}
waitFrame(t, frames, "permission", func(frame protocol.ChatEventFrame) bool {
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.Name == "Write"
})
var pending []protocol.ChatPermissionFrame
eventually(t, "pending approval", func() bool {
pending, _ = chat.Permissions(context.Background(), session.ID)
return len(pending) == 1
})
if err := chat.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: pending[0].RequestID, Decision: protocol.ChatDecisionOnce}); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
eventually(t, "transcript", func() bool { return sessionTranscriptPath(session.ID) != "" })
page, err := chat.Messages(context.Background(), session.ID, 50, "")
if err != nil {
t.Fatal(err)
}
if len(page.Messages) != 2 {
t.Fatalf("history messages: %+v", page.Messages)
}
user := page.Messages[0]
if user.Role != protocol.ChatRoleUser || user.ID != "native-user-1" {
t.Fatalf("user message identity: %+v", user)
}
if len(user.Parts) != 1 || user.Parts[0].Text != "permission please" {
t.Fatalf("user parts: %+v", user.Parts)
}
assistant := page.Messages[1]
if assistant.Role != protocol.ChatRoleAssistant || len(assistant.Parts) != 2 {
t.Fatalf("assistant message: %+v", assistant)
}
if assistant.Parts[0].Type != protocol.ChatPartText || assistant.Parts[0].Text != "Hello world" {
t.Fatalf("assistant text: %+v", assistant.Parts[0])
}
tool := assistant.Parts[1]
if tool.Type != protocol.ChatPartTool || tool.Name != "Write" || tool.State == nil || tool.State.Status != protocol.ChatToolCompleted {
t.Fatalf("assistant tool: %+v", tool)
}
if len(tool.State.Content) != 1 || !strings.Contains(tool.State.Content[0].Text, "File created") {
t.Fatalf("tool content: %+v", tool.State.Content)
}
// Newest-first paging: one long assistant turn is one message.
first, err := chat.Messages(context.Background(), session.ID, 1, "")
if err != nil {
t.Fatal(err)
}
if len(first.Messages) != 1 || first.Messages[0].Role != protocol.ChatRoleAssistant || !first.HasMore || first.NextCursor == "" {
t.Fatalf("first page: %+v", first)
}
older, err := chat.Messages(context.Background(), session.ID, 1, first.NextCursor)
if err != nil {
t.Fatal(err)
}
if len(older.Messages) != 1 || older.Messages[0].Role != protocol.ChatRoleUser || older.HasMore {
t.Fatalf("older page: %+v", older)
}
}
func TestAttachmentContentIncludesImageAndText(t *testing.T) {
chat := testChat(t)
session := newCreatedChat(t, chat)
dir := t.TempDir()
imagePath := filepath.Join(dir, "pixel.png")
imageData := []byte{0x89, 0x50, 0x4e, 0x47, 0, 1, 2, 3}
if err := os.WriteFile(imagePath, imageData, 0o600); err != nil {
t.Fatal(err)
}
textPath := filepath.Join(dir, "notes.txt")
if err := os.WriteFile(textPath, []byte("fixture notes"), 0o600); err != nil {
t.Fatal(err)
}
frames := subscribe(t, chat, session.ID)
files := []opencode.PromptFile{
opencode.NewPreparedPromptFile("file://"+imagePath, "pixel.png"),
opencode.NewPreparedPromptFile("file://"+textPath, "notes.txt"),
}
if err := chat.PromptWithMessageIDFiles(context.Background(), session.ID, "see attached", "native-user-2", files); err != nil {
t.Fatal(err)
}
waitStepEnded(t, frames)
content, err := os.ReadFile(os.Getenv("FAKE_CLAUDE_LOG") + ".content")
if err != nil {
t.Fatal(err)
}
var blocks []map[string]any
if err := json.Unmarshal([]byte(strings.TrimSpace(string(content))), &blocks); err != nil {
t.Fatal(err)
}
var sawImage, sawText bool
for _, block := range blocks {
if block["type"] == "image" {
source, _ := block["source"].(map[string]any)
decoded, _ := base64.StdEncoding.DecodeString(source["data"].(string))
if string(decoded) != string(imageData) || source["media_type"] != "image/png" {
t.Fatalf("image block: %+v", block)
}
sawImage = true
}
if block["type"] == "text" && strings.Contains(block["text"].(string), "fixture notes") {
sawText = true
}
}
if !sawImage || !sawText {
t.Fatalf("missing attachment content: %+v", blocks)
}
}
func TestSignedOutCLIReturnsAnActionableError(t *testing.T) {
chat := testChat(t)
session := newCreatedChat(t, chat)
t.Setenv("FAKE_CLAUDE_FAIL", "1")
err := chat.Prompt(context.Background(), session.ID, "hello")
if err == nil {
t.Fatal("a signed-out CLI must fail the prompt")
}
if !strings.Contains(strings.ToLower(err.Error()), "login") && !strings.Contains(strings.ToLower(err.Error()), "sign") {
t.Fatalf("error is not actionable: %v", err)
}
}