Files
pipelines/internal/server/chat_browser_test.go
T

219 lines
8.2 KiB
Go

package server
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type browserModelFixture struct {
*fakeChat
forms *browserForms
}
type browserForms struct {
mu sync.Mutex
pending []protocol.ChatForm
answered int
canceled int
outputReads int
}
func (b browserModelFixture) Forms(context.Context, string) ([]protocol.ChatForm, error) {
b.forms.mu.Lock()
defer b.forms.mu.Unlock()
return append([]protocol.ChatForm(nil), b.forms.pending...), nil
}
func (b browserModelFixture) Answer(_ context.Context, answer protocol.ChatAnswerRequest) error {
b.forms.mu.Lock()
defer b.forms.mu.Unlock()
if answer.Action != "cancel" && answer.Answers["answer"] != "Browser answer" {
return fmt.Errorf("unexpected answer")
}
if answer.FormID == "form_second" {
if answer.Answers["enabled"] != false || answer.Answers["count"] != float64(0) {
return fmt.Errorf("typed answers were lost")
}
if _, ok := answer.Answers["conditional"]; ok {
return fmt.Errorf("hidden conditional answer leaked")
}
}
for i, form := range b.forms.pending {
if form.ID == answer.FormID {
b.forms.pending = append(b.forms.pending[:i], b.forms.pending[i+1:]...)
if answer.Action == "cancel" {
b.forms.canceled++
} else {
b.forms.answered++
}
return nil
}
}
return fmt.Errorf("unknown form")
}
func (b browserModelFixture) Reply(ctx context.Context, answer protocol.ChatReplyRequest) error {
if err := b.fakeChat.Reply(ctx, answer); err != nil {
return err
}
b.fakeChat.mu.Lock()
defer b.fakeChat.mu.Unlock()
for i, p := range b.fakeChat.permissions {
if p.RequestID == answer.RequestID {
b.fakeChat.permissions = append(b.fakeChat.permissions[:i], b.fakeChat.permissions[i+1:]...)
break
}
}
return nil
}
func (b browserModelFixture) ShellOutput(_ context.Context, q protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) {
b.fakeChat.mu.Lock()
defer b.fakeChat.mu.Unlock()
for _, shell := range b.fakeChat.shells {
if shell.ID == q.ShellID && q.SessionID == b.fakeChat.sessions[0].ID {
b.forms.mu.Lock()
b.forms.outputReads++
b.forms.mu.Unlock()
return protocol.ChatShellOutputPage{Shell: shell, Output: "Controlled background output", Size: 28}, nil
}
}
return protocol.ChatShellOutputPage{}, fmt.Errorf("unknown task")
}
func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(250 * time.Millisecond):
}
return b.fakeChat.Models(ctx, id)
}
func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
}
for _, backend := range []string{"opencode", "codex"} {
t.Run(backend, func(t *testing.T) { checkChatBrowserLifecycle(t, backend) })
}
}
func checkChatBrowserLifecycle(t *testing.T, backend string) {
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
directory := t.TempDir()
session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory}
session.Backend = backend
if backend == "codex" {
session.ID = "01900000-0000-7000-8000-000000000001"
}
if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
fake := scriptedChat()
fake.block = true
fake.sessions = []protocol.ChatSession{session}
for i := 0; i < 30; i++ {
other := protocol.ChatSession{ID: fmt.Sprintf("ses_browser_%d", i), Title: fmt.Sprintf("Scratch chat %d", i), Directory: directory}
if err := store.RecordOperatorSession(ctx, other.ID, directory, other.Title, protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
fake.sessions = append(fake.sessions, other)
}
fake.models = []protocol.ChatModel{{ID: "model-a", Provider: "fixture", Name: "Model Alpha"}, {ID: "model-b", Provider: "fixture", Name: "Model Beta"}}
fake.messages = []protocol.ChatMessage{
{ID: "msg_user", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: "text", Text: strings.Repeat("Earlier scratch prompt line\n", 100)}}},
{ID: "msg_answer", Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{{Type: "reasoning", Text: "Controlled reasoning evidence"}, {Type: "text", Text: "Controlled answer"}}},
}
forms := &browserForms{}
handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake, forms}}, store)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/fixture-resync" && req.Method == http.MethodPost {
fake.mu.Lock()
fake.messages = append(fake.messages, protocol.ChatMessage{ID: "msg_resync", Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Authoritative resync evidence"}}})
fake.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
return
}
if req.URL.Path == "/fixture-shell" && req.Method == http.MethodPost {
fake.mu.Lock()
fake.shells = []protocol.ChatShell{{ID: "sh_" + backend + "_fixture", Status: protocol.ChatShellRunning, Command: "printf controlled"}}
fake.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
return
}
if req.URL.Path == "/fixture-cancel" && req.Method == http.MethodPost {
forms.mu.Lock()
forms.pending = append(forms.pending, protocol.ChatForm{ID: "form_cancel", SessionID: session.ID, Title: "Cancel this question", Actions: []string{"accept", "cancel"}, Fields: []protocol.ChatFormField{{Key: "answer", Type: "string", Required: true}}})
forms.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
return
}
if req.URL.Path == "/fixture-attention" && req.Method == http.MethodPost {
fake.mu.Lock()
fake.permissions = []protocol.ChatPermissionFrame{
{SessionID: session.ID, RequestID: "per_first", Action: "shell", Resources: []string{"Controlled first command"}},
{SessionID: session.ID, RequestID: "per_second", Action: "edit", Resources: []string{"Controlled second path"}},
}
fake.mu.Unlock()
forms.mu.Lock()
for _, id := range []string{"form_first", "form_second"} {
fields := []protocol.ChatFormField{{Key: "answer", Type: "string", Title: "Your answer", Required: true}}
if id == "form_second" {
fields = append(fields,
protocol.ChatFormField{Key: "enabled", Type: "boolean", Title: "Enabled", Required: true},
protocol.ChatFormField{Key: "count", Type: "integer", Title: "Count", Minimum: 0, Maximum: 5, Required: true},
protocol.ChatFormField{Key: "secret", Type: "string", Title: "Secret", Secret: true},
protocol.ChatFormField{Key: "conditional", Type: "string", Title: "Only when enabled", Required: true, When: []protocol.ChatFormWhen{{Key: "enabled", Op: "eq", Value: true}}},
)
}
forms.pending = append(forms.pending, protocol.ChatForm{ID: id, SessionID: session.ID, Title: "Recovered question", Fields: fields})
}
forms.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
return
}
handler.ServeHTTP(w, req)
}))
defer srv.Close()
cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID)
if captures := os.Getenv("PIPELINES_CAPTURE_DIR"); captures != "" {
cmd.Env = append(os.Environ(), "PIPELINES_CAPTURE_DIR="+filepath.Join(captures, backend))
}
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("chat browser regression: %v\n%s", err, output)
} else {
t.Log(string(output))
}
fake.mu.Lock()
replies := fake.replies
fake.mu.Unlock()
forms.mu.Lock()
answers := forms.answered
canceled := forms.canceled
outputReads := forms.outputReads
forms.mu.Unlock()
if replies != 2 || answers != 2 || canceled != 1 || outputReads != 1 {
t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d output=%d", replies, answers, canceled, outputReads)
}
}