459 lines
15 KiB
Go
459 lines
15 KiB
Go
package claude
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/opencode"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func testChat(t *testing.T) *Chat {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
source, err := os.ReadFile(filepath.Join("testdata", "fake-claude.py"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
script := filepath.Join(dir, "claude")
|
|
if err := os.WriteFile(script, source, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PIPELINES_CLAUDE", script)
|
|
t.Setenv("CLAUDE_CONFIG_DIR", filepath.Join(dir, "config"))
|
|
t.Setenv("FAKE_CLAUDE_LOG", filepath.Join(dir, "args.log"))
|
|
chat := New(context.Background())
|
|
t.Cleanup(chat.Close)
|
|
return chat
|
|
}
|
|
|
|
func newCreatedChat(t *testing.T, chat *Chat) protocol.ChatSession {
|
|
t.Helper()
|
|
session, err := chat.Create(context.Background(), "Fixture", t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return session
|
|
}
|
|
|
|
func subscribe(t *testing.T, chat *Chat, id string) chan protocol.ChatEventFrame {
|
|
t.Helper()
|
|
frames := make(chan protocol.ChatEventFrame, 512)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
_ = chat.Stream(ctx, id, nil, func(frame protocol.ChatEventFrame) error {
|
|
select {
|
|
case frames <- frame:
|
|
default:
|
|
}
|
|
return nil
|
|
})
|
|
}()
|
|
t.Cleanup(func() {
|
|
cancel()
|
|
<-done
|
|
})
|
|
// Wait for the subscription to be attached before prompting.
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
chat.session(id).mu.Lock()
|
|
count := len(chat.session(id).subs)
|
|
chat.session(id).mu.Unlock()
|
|
if count > 0 {
|
|
return frames
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
t.Fatal("stream did not attach")
|
|
return nil
|
|
}
|
|
|
|
func waitFrame(t *testing.T, frames <-chan protocol.ChatEventFrame, what string, match func(protocol.ChatEventFrame) bool) protocol.ChatEventFrame {
|
|
t.Helper()
|
|
deadline := time.After(6 * time.Second)
|
|
for {
|
|
select {
|
|
case frame := <-frames:
|
|
if match(frame) {
|
|
return frame
|
|
}
|
|
case <-deadline:
|
|
t.Fatalf("timed out waiting for %s", what)
|
|
}
|
|
}
|
|
}
|
|
|
|
func waitStepEnded(t *testing.T, frames <-chan protocol.ChatEventFrame) {
|
|
t.Helper()
|
|
waitFrame(t, frames, "step ended", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded
|
|
})
|
|
}
|
|
|
|
func eventually(t *testing.T, what string, check func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(6 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if check() {
|
|
return
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatalf("timed out waiting for %s", what)
|
|
}
|
|
|
|
func TestTurnStreamsTextWithoutDuplicatingAndReportsUsage(t *testing.T) {
|
|
chat := testChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
frames := subscribe(t, chat, session.ID)
|
|
if err := chat.Prompt(context.Background(), session.ID, "hello"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := ""
|
|
var usage *protocol.ChatUsage
|
|
ended := false
|
|
deadline := time.After(8 * time.Second)
|
|
for !ended || usage == nil {
|
|
select {
|
|
case frame := <-frames:
|
|
switch frame.Kind {
|
|
case protocol.ChatPartText:
|
|
text += frame.Delta
|
|
case protocol.ChatEventStep:
|
|
if frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded {
|
|
ended = true
|
|
}
|
|
case protocol.ChatEventUsage:
|
|
usage = frame.Usage
|
|
}
|
|
case <-deadline:
|
|
t.Fatalf("turn did not settle (ended=%v usage=%v text=%q)", ended, usage != nil, text)
|
|
}
|
|
}
|
|
if text != "Hello world" {
|
|
t.Fatalf("streamed text = %q, want one copy of Hello world", text)
|
|
}
|
|
if usage.TotalTokens == nil || *usage.TotalTokens != 100 {
|
|
t.Fatalf("usage totals: %+v", usage)
|
|
}
|
|
if usage.ContextWindow == nil || *usage.ContextWindow != 200000 || usage.ContextTokens == nil || *usage.ContextTokens != 60 {
|
|
t.Fatalf("context usage: %+v", usage)
|
|
}
|
|
if usage.CostUSD == nil || *usage.CostUSD != 0.25 || usage.Scope != "session" {
|
|
t.Fatalf("cost/scope: %+v", usage)
|
|
}
|
|
}
|
|
|
|
func TestApprovalAllowAndRejectReachTheCLI(t *testing.T) {
|
|
chat := testChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
frames := subscribe(t, chat, session.ID)
|
|
if err := chat.Prompt(context.Background(), session.ID, "permission please"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "running tool", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.Name == "Write" && frame.Part.State != nil && frame.Part.State.Status == protocol.ChatToolRunning
|
|
})
|
|
var pending []protocol.ChatPermissionFrame
|
|
eventually(t, "permission request", func() bool {
|
|
var err error
|
|
pending, err = chat.Permissions(context.Background(), session.ID)
|
|
return err == nil && len(pending) == 1
|
|
})
|
|
request := pending[0]
|
|
if request.Action != "Write" || len(request.Resources) != 1 || !strings.HasSuffix(request.Resources[0], "note.txt") {
|
|
t.Fatalf("approval frame: %+v", request)
|
|
}
|
|
if request.AlwaysLabel == "" {
|
|
t.Fatal("a suggestion should offer Always allow")
|
|
}
|
|
if err := chat.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: request.RequestID, Decision: protocol.ChatDecisionOnce}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "completed tool", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.ID == "toolu_fake_1" &&
|
|
frame.Part.State != nil && frame.Part.State.Status == protocol.ChatToolCompleted &&
|
|
len(frame.Part.State.Content) == 1 && strings.Contains(frame.Part.State.Content[0].Text, "File created")
|
|
})
|
|
waitStepEnded(t, frames)
|
|
|
|
// A rejection is delivered as a deny, and the turn still settles.
|
|
if err := chat.Prompt(context.Background(), session.ID, "permission reject"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
eventually(t, "second permission request", func() bool {
|
|
var err error
|
|
pending, err = chat.Permissions(context.Background(), session.ID)
|
|
return err == nil && len(pending) == 1
|
|
})
|
|
if err := chat.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: pending[0].RequestID, Decision: protocol.ChatDecisionReject}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "rejected tool", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
|
|
frame.Part.State.Status == protocol.ChatToolError && frame.Part.State.Error != nil
|
|
})
|
|
waitStepEnded(t, frames)
|
|
if remaining, _ := chat.Permissions(context.Background(), session.ID); len(remaining) != 0 {
|
|
t.Fatalf("resolved approval still pending: %+v", remaining)
|
|
}
|
|
}
|
|
|
|
func TestQuestionBecomesAFormAndAnswerReturnsToCLI(t *testing.T) {
|
|
chat := testChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
frames := subscribe(t, chat, session.ID)
|
|
if err := chat.Prompt(context.Background(), session.ID, "question please"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var forms []protocol.ChatForm
|
|
eventually(t, "question form", func() bool {
|
|
var err error
|
|
forms, err = chat.Forms(context.Background(), session.ID)
|
|
return err == nil && len(forms) == 1
|
|
})
|
|
form := forms[0]
|
|
if form.Kind != "question" || len(form.Fields) != 1 || len(form.Fields[0].Options) != 2 {
|
|
t.Fatalf("form: %+v", form)
|
|
}
|
|
if err := chat.Answer(context.Background(), protocol.ChatAnswerRequest{SessionID: session.ID, FormID: form.ID, Answers: map[string]any{"q0": "Alpha"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "answered tool", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.State != nil &&
|
|
frame.Part.State.Status == protocol.ChatToolCompleted && len(frame.Part.State.Content) == 1 &&
|
|
strings.Contains(frame.Part.State.Content[0].Text, "Alpha")
|
|
})
|
|
waitStepEnded(t, frames)
|
|
if forms, _ := chat.Forms(context.Background(), session.ID); len(forms) != 0 {
|
|
t.Fatalf("resolved form still pending: %+v", forms)
|
|
}
|
|
}
|
|
|
|
func TestInterruptSettlesTheTurn(t *testing.T) {
|
|
chat := testChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
frames := subscribe(t, chat, session.ID)
|
|
if err := chat.Prompt(context.Background(), session.ID, "long task"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := chat.Interrupt(context.Background(), session.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
chat.session(session.ID).mu.Lock()
|
|
running := chat.session(session.ID).running
|
|
chat.session(session.ID).mu.Unlock()
|
|
if running {
|
|
t.Fatal("session still marked running after interrupt")
|
|
}
|
|
}
|
|
|
|
func TestModelChoiceIsAppliedAndProcessResumes(t *testing.T) {
|
|
chat := testChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
if err := chat.SwitchModel(context.Background(), session.ID, Backend, "sonnet"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
models, err := chat.Models(context.Background(), session.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(models) != 3 || models[1].ID != "sonnet" || models[1].Provider != Backend {
|
|
t.Fatalf("models: %+v", models)
|
|
}
|
|
frames := subscribe(t, chat, session.ID)
|
|
if err := chat.Prompt(context.Background(), session.ID, "hello"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
logPath := os.Getenv("FAKE_CLAUDE_LOG")
|
|
entries := readArgs(t, logPath)
|
|
if argValue(entries[len(entries)-1].Args, "--model") != "sonnet" {
|
|
t.Fatalf("spawn did not apply the selected model: %+v", entries)
|
|
}
|
|
if argValue(entries[len(entries)-1].Args, "--session-id") != session.ID {
|
|
t.Fatalf("new session did not pin its id: %+v", entries)
|
|
}
|
|
// Releasing the process and prompting again must resume the same session.
|
|
chat.session(session.ID).closeProcess()
|
|
if err := chat.Prompt(context.Background(), session.ID, "hello again"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
entries = readArgs(t, logPath)
|
|
if len(entries) < 2 || argValue(entries[len(entries)-1].Args, "--resume") != session.ID {
|
|
t.Fatalf("restart did not resume the session: %+v", entries)
|
|
}
|
|
}
|
|
|
|
type argsEntry struct {
|
|
Args []string `json:"args"`
|
|
Cwd string `json:"cwd"`
|
|
}
|
|
|
|
func readArgs(t *testing.T, path string) []argsEntry {
|
|
t.Helper()
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var entries []argsEntry
|
|
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var entry argsEntry
|
|
if json.Unmarshal([]byte(line), &entry) == nil {
|
|
entries = append(entries, entry)
|
|
}
|
|
}
|
|
return entries
|
|
}
|
|
|
|
func argValue(values []string, flag string) string {
|
|
for index, value := range values {
|
|
if value == flag && index+1 < len(values) {
|
|
return values[index+1]
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func TestHistoryMergesBlocksAndMapsPromptIdentity(t *testing.T) {
|
|
chat := testChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
frames := subscribe(t, chat, session.ID)
|
|
if err := chat.PromptWithMessageID(context.Background(), session.ID, "permission please", "native-user-1"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFrame(t, frames, "permission", func(frame protocol.ChatEventFrame) bool {
|
|
return frame.Kind == protocol.ChatEventTool && frame.Part != nil && frame.Part.Name == "Write"
|
|
})
|
|
var pending []protocol.ChatPermissionFrame
|
|
eventually(t, "pending approval", func() bool {
|
|
pending, _ = chat.Permissions(context.Background(), session.ID)
|
|
return len(pending) == 1
|
|
})
|
|
if err := chat.Reply(context.Background(), protocol.ChatReplyRequest{SessionID: session.ID, RequestID: pending[0].RequestID, Decision: protocol.ChatDecisionOnce}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
eventually(t, "transcript", func() bool { return sessionTranscriptPath(session.ID) != "" })
|
|
|
|
page, err := chat.Messages(context.Background(), session.ID, 50, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(page.Messages) != 2 {
|
|
t.Fatalf("history messages: %+v", page.Messages)
|
|
}
|
|
user := page.Messages[0]
|
|
if user.Role != protocol.ChatRoleUser || user.ID != "native-user-1" {
|
|
t.Fatalf("user message identity: %+v", user)
|
|
}
|
|
if len(user.Parts) != 1 || user.Parts[0].Text != "permission please" {
|
|
t.Fatalf("user parts: %+v", user.Parts)
|
|
}
|
|
assistant := page.Messages[1]
|
|
if assistant.Role != protocol.ChatRoleAssistant || len(assistant.Parts) != 2 {
|
|
t.Fatalf("assistant message: %+v", assistant)
|
|
}
|
|
if assistant.Parts[0].Type != protocol.ChatPartText || assistant.Parts[0].Text != "Hello world" {
|
|
t.Fatalf("assistant text: %+v", assistant.Parts[0])
|
|
}
|
|
tool := assistant.Parts[1]
|
|
if tool.Type != protocol.ChatPartTool || tool.Name != "Write" || tool.State == nil || tool.State.Status != protocol.ChatToolCompleted {
|
|
t.Fatalf("assistant tool: %+v", tool)
|
|
}
|
|
if len(tool.State.Content) != 1 || !strings.Contains(tool.State.Content[0].Text, "File created") {
|
|
t.Fatalf("tool content: %+v", tool.State.Content)
|
|
}
|
|
|
|
// Newest-first paging: one long assistant turn is one message.
|
|
first, err := chat.Messages(context.Background(), session.ID, 1, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(first.Messages) != 1 || first.Messages[0].Role != protocol.ChatRoleAssistant || !first.HasMore || first.NextCursor == "" {
|
|
t.Fatalf("first page: %+v", first)
|
|
}
|
|
older, err := chat.Messages(context.Background(), session.ID, 1, first.NextCursor)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(older.Messages) != 1 || older.Messages[0].Role != protocol.ChatRoleUser || older.HasMore {
|
|
t.Fatalf("older page: %+v", older)
|
|
}
|
|
}
|
|
|
|
func TestAttachmentContentIncludesImageAndText(t *testing.T) {
|
|
chat := testChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
dir := t.TempDir()
|
|
imagePath := filepath.Join(dir, "pixel.png")
|
|
imageData := []byte{0x89, 0x50, 0x4e, 0x47, 0, 1, 2, 3}
|
|
if err := os.WriteFile(imagePath, imageData, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
textPath := filepath.Join(dir, "notes.txt")
|
|
if err := os.WriteFile(textPath, []byte("fixture notes"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
frames := subscribe(t, chat, session.ID)
|
|
files := []opencode.PromptFile{
|
|
opencode.NewPreparedPromptFile("file://"+imagePath, "pixel.png"),
|
|
opencode.NewPreparedPromptFile("file://"+textPath, "notes.txt"),
|
|
}
|
|
if err := chat.PromptWithMessageIDFiles(context.Background(), session.ID, "see attached", "native-user-2", files); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitStepEnded(t, frames)
|
|
content, err := os.ReadFile(os.Getenv("FAKE_CLAUDE_LOG") + ".content")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var blocks []map[string]any
|
|
if err := json.Unmarshal([]byte(strings.TrimSpace(string(content))), &blocks); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var sawImage, sawText bool
|
|
for _, block := range blocks {
|
|
if block["type"] == "image" {
|
|
source, _ := block["source"].(map[string]any)
|
|
decoded, _ := base64.StdEncoding.DecodeString(source["data"].(string))
|
|
if string(decoded) != string(imageData) || source["media_type"] != "image/png" {
|
|
t.Fatalf("image block: %+v", block)
|
|
}
|
|
sawImage = true
|
|
}
|
|
if block["type"] == "text" && strings.Contains(block["text"].(string), "fixture notes") {
|
|
sawText = true
|
|
}
|
|
}
|
|
if !sawImage || !sawText {
|
|
t.Fatalf("missing attachment content: %+v", blocks)
|
|
}
|
|
}
|
|
|
|
func TestSignedOutCLIReturnsAnActionableError(t *testing.T) {
|
|
chat := testChat(t)
|
|
session := newCreatedChat(t, chat)
|
|
t.Setenv("FAKE_CLAUDE_FAIL", "1")
|
|
err := chat.Prompt(context.Background(), session.ID, "hello")
|
|
if err == nil {
|
|
t.Fatal("a signed-out CLI must fail the prompt")
|
|
}
|
|
if !strings.Contains(strings.ToLower(err.Error()), "login") && !strings.Contains(strings.ToLower(err.Error()), "sign") {
|
|
t.Fatalf("error is not actionable: %v", err)
|
|
}
|
|
}
|