219 lines
8.2 KiB
Go
219 lines
8.2 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
type browserModelFixture struct {
|
|
*fakeChat
|
|
forms *browserForms
|
|
}
|
|
|
|
type browserForms struct {
|
|
mu sync.Mutex
|
|
pending []protocol.ChatForm
|
|
answered int
|
|
canceled int
|
|
outputReads int
|
|
}
|
|
|
|
func (b browserModelFixture) Forms(context.Context, string) ([]protocol.ChatForm, error) {
|
|
b.forms.mu.Lock()
|
|
defer b.forms.mu.Unlock()
|
|
return append([]protocol.ChatForm(nil), b.forms.pending...), nil
|
|
}
|
|
|
|
func (b browserModelFixture) Answer(_ context.Context, answer protocol.ChatAnswerRequest) error {
|
|
b.forms.mu.Lock()
|
|
defer b.forms.mu.Unlock()
|
|
if answer.Action != "cancel" && answer.Answers["answer"] != "Browser answer" {
|
|
return fmt.Errorf("unexpected answer")
|
|
}
|
|
if answer.FormID == "form_second" {
|
|
if answer.Answers["enabled"] != false || answer.Answers["count"] != float64(0) {
|
|
return fmt.Errorf("typed answers were lost")
|
|
}
|
|
if _, ok := answer.Answers["conditional"]; ok {
|
|
return fmt.Errorf("hidden conditional answer leaked")
|
|
}
|
|
}
|
|
for i, form := range b.forms.pending {
|
|
if form.ID == answer.FormID {
|
|
b.forms.pending = append(b.forms.pending[:i], b.forms.pending[i+1:]...)
|
|
if answer.Action == "cancel" {
|
|
b.forms.canceled++
|
|
} else {
|
|
b.forms.answered++
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
return fmt.Errorf("unknown form")
|
|
}
|
|
|
|
func (b browserModelFixture) Reply(ctx context.Context, answer protocol.ChatReplyRequest) error {
|
|
if err := b.fakeChat.Reply(ctx, answer); err != nil {
|
|
return err
|
|
}
|
|
b.fakeChat.mu.Lock()
|
|
defer b.fakeChat.mu.Unlock()
|
|
for i, p := range b.fakeChat.permissions {
|
|
if p.RequestID == answer.RequestID {
|
|
b.fakeChat.permissions = append(b.fakeChat.permissions[:i], b.fakeChat.permissions[i+1:]...)
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b browserModelFixture) ShellOutput(_ context.Context, q protocol.ChatShellOutputQuery) (protocol.ChatShellOutputPage, error) {
|
|
b.fakeChat.mu.Lock()
|
|
defer b.fakeChat.mu.Unlock()
|
|
for _, shell := range b.fakeChat.shells {
|
|
if shell.ID == q.ShellID && q.SessionID == b.fakeChat.sessions[0].ID {
|
|
b.forms.mu.Lock()
|
|
b.forms.outputReads++
|
|
b.forms.mu.Unlock()
|
|
return protocol.ChatShellOutputPage{Shell: shell, Output: "Controlled background output", Size: 28}, nil
|
|
}
|
|
}
|
|
return protocol.ChatShellOutputPage{}, fmt.Errorf("unknown task")
|
|
}
|
|
|
|
func (b browserModelFixture) Models(ctx context.Context, id string) ([]protocol.ChatModel, error) {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-time.After(250 * time.Millisecond):
|
|
}
|
|
return b.fakeChat.Models(ctx, id)
|
|
}
|
|
|
|
func TestChatBrowserLifecycleAndModelDialog(t *testing.T) {
|
|
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
|
|
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
|
|
}
|
|
for _, backend := range []string{"opencode", "codex"} {
|
|
t.Run(backend, func(t *testing.T) { checkChatBrowserLifecycle(t, backend) })
|
|
}
|
|
}
|
|
|
|
func checkChatBrowserLifecycle(t *testing.T, backend string) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "scratch.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
directory := t.TempDir()
|
|
session := protocol.ChatSession{ID: "ses_browser", Title: "Scratch chat", Model: "model-a", Directory: directory}
|
|
session.Backend = backend
|
|
if backend == "codex" {
|
|
session.ID = "01900000-0000-7000-8000-000000000001"
|
|
}
|
|
if err := store.RecordOperatorSession(ctx, session.ID, directory, session.Title, protocol.ChatSessionTagOperator); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fake := scriptedChat()
|
|
fake.block = true
|
|
fake.sessions = []protocol.ChatSession{session}
|
|
for i := 0; i < 30; i++ {
|
|
other := protocol.ChatSession{ID: fmt.Sprintf("ses_browser_%d", i), Title: fmt.Sprintf("Scratch chat %d", i), Directory: directory}
|
|
if err := store.RecordOperatorSession(ctx, other.ID, directory, other.Title, protocol.ChatSessionTagOperator); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fake.sessions = append(fake.sessions, other)
|
|
}
|
|
fake.models = []protocol.ChatModel{{ID: "model-a", Provider: "fixture", Name: "Model Alpha"}, {ID: "model-b", Provider: "fixture", Name: "Model Beta"}}
|
|
fake.messages = []protocol.ChatMessage{
|
|
{ID: "msg_user", Role: protocol.ChatRoleUser, Parts: []protocol.ChatPart{{Type: "text", Text: strings.Repeat("Earlier scratch prompt line\n", 100)}}},
|
|
{ID: "msg_answer", Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{{Type: "reasoning", Text: "Controlled reasoning evidence"}, {Type: "text", Text: "Controlled answer"}}},
|
|
}
|
|
forms := &browserForms{}
|
|
handler := HandlerWithOptions(ctx, Options{Chat: browserModelFixture{fake, forms}}, store)
|
|
defer func() { cancel(); WaitRuns(handler) }()
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
if req.URL.Path == "/fixture-resync" && req.Method == http.MethodPost {
|
|
fake.mu.Lock()
|
|
fake.messages = append(fake.messages, protocol.ChatMessage{ID: "msg_resync", Role: protocol.ChatRoleAssistant, Parts: []protocol.ChatPart{{Type: protocol.ChatPartText, Text: "Authoritative resync evidence"}}})
|
|
fake.mu.Unlock()
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if req.URL.Path == "/fixture-shell" && req.Method == http.MethodPost {
|
|
fake.mu.Lock()
|
|
fake.shells = []protocol.ChatShell{{ID: "sh_" + backend + "_fixture", Status: protocol.ChatShellRunning, Command: "printf controlled"}}
|
|
fake.mu.Unlock()
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if req.URL.Path == "/fixture-cancel" && req.Method == http.MethodPost {
|
|
forms.mu.Lock()
|
|
forms.pending = append(forms.pending, protocol.ChatForm{ID: "form_cancel", SessionID: session.ID, Title: "Cancel this question", Actions: []string{"accept", "cancel"}, Fields: []protocol.ChatFormField{{Key: "answer", Type: "string", Required: true}}})
|
|
forms.mu.Unlock()
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if req.URL.Path == "/fixture-attention" && req.Method == http.MethodPost {
|
|
fake.mu.Lock()
|
|
fake.permissions = []protocol.ChatPermissionFrame{
|
|
{SessionID: session.ID, RequestID: "per_first", Action: "shell", Resources: []string{"Controlled first command"}},
|
|
{SessionID: session.ID, RequestID: "per_second", Action: "edit", Resources: []string{"Controlled second path"}},
|
|
}
|
|
fake.mu.Unlock()
|
|
forms.mu.Lock()
|
|
for _, id := range []string{"form_first", "form_second"} {
|
|
fields := []protocol.ChatFormField{{Key: "answer", Type: "string", Title: "Your answer", Required: true}}
|
|
if id == "form_second" {
|
|
fields = append(fields,
|
|
protocol.ChatFormField{Key: "enabled", Type: "boolean", Title: "Enabled", Required: true},
|
|
protocol.ChatFormField{Key: "count", Type: "integer", Title: "Count", Minimum: 0, Maximum: 5, Required: true},
|
|
protocol.ChatFormField{Key: "secret", Type: "string", Title: "Secret", Secret: true},
|
|
protocol.ChatFormField{Key: "conditional", Type: "string", Title: "Only when enabled", Required: true, When: []protocol.ChatFormWhen{{Key: "enabled", Op: "eq", Value: true}}},
|
|
)
|
|
}
|
|
forms.pending = append(forms.pending, protocol.ChatForm{ID: id, SessionID: session.ID, Title: "Recovered question", Fields: fields})
|
|
}
|
|
forms.mu.Unlock()
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
handler.ServeHTTP(w, req)
|
|
}))
|
|
defer srv.Close()
|
|
cmd := exec.CommandContext(ctx, "node", "../../web/test/chat-hardening-probe.mjs", strings.TrimPrefix(srv.URL, "http://"), session.ID)
|
|
if captures := os.Getenv("PIPELINES_CAPTURE_DIR"); captures != "" {
|
|
cmd.Env = append(os.Environ(), "PIPELINES_CAPTURE_DIR="+filepath.Join(captures, backend))
|
|
}
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("chat browser regression: %v\n%s", err, output)
|
|
} else {
|
|
t.Log(string(output))
|
|
}
|
|
fake.mu.Lock()
|
|
replies := fake.replies
|
|
fake.mu.Unlock()
|
|
forms.mu.Lock()
|
|
answers := forms.answered
|
|
canceled := forms.canceled
|
|
outputReads := forms.outputReads
|
|
forms.mu.Unlock()
|
|
if replies != 2 || answers != 2 || canceled != 1 || outputReads != 1 {
|
|
t.Fatalf("browser did not deliver all interactions: approvals=%d answers=%d canceled=%d output=%d", replies, answers, canceled, outputReads)
|
|
}
|
|
}
|